epub-translator 0.0.7__py3-none-any.whl → 0.1.1__py3-none-any.whl

epub_translator/epub/html/file.py

@@ -1,80 +0,0 @@
-import re
-
-from typing import Iterable
-from xml.etree.ElementTree import fromstring, tostring, Element
-from .dom_operator import read_texts, write_texts
-from .empty_tags import to_xml, to_html
-
-
-_FILE_HEAD_PATTERN = re.compile(r"^<\?xml.*?\?>[\s]*<!DOCTYPE.*?>")
-_XMLNS_IN_TAG = re.compile(r"\{[^}]+\}")
-_BRACES = re.compile(r"(\{|\})")
-
-class HTMLFile:
-  def __init__(self, file_content: str):
-    match = re.match(_FILE_HEAD_PATTERN, file_content)
-    xml_content = re.sub(_FILE_HEAD_PATTERN, "", to_xml(file_content))
-    self._head: str = match.group() if match else None
-    self._root: Element = fromstring(xml_content)
-    self._xmlns: str | None = self._extract_xmlns(self._root)
-    self._texts_length: int | None = None
-
-  def _extract_xmlns(self, root: Element) -> str | None:
-    root_xmlns: str | None = None
-    for i, element in enumerate(_all_elements(root)):
-      need_clean_xmlns = True
-      match = re.match(_XMLNS_IN_TAG, element.tag)
-
-      if match:
-        xmlns = re.sub(_BRACES, "", match.group())
-        if i == 0:
-          root_xmlns = xmlns
-        elif root_xmlns != xmlns:
-          need_clean_xmlns = False
-      if need_clean_xmlns:
-        element.tag = re.sub(_XMLNS_IN_TAG, "", element.tag)
-
-    return root_xmlns
-
-  def read_texts(self) -> list[str]:
-    texts = list(read_texts(self._root))
-    self._texts_length = len(texts)
-    return texts
-
-  def write_texts(self, texts: Iterable[str], append: bool):
-    write_texts(self._root, texts, append)
-
-  @property
-  def texts_length(self) -> int:
-    if self._texts_length is None:
-      self._texts_length = 0
-      for _ in read_texts(self._root):
-        self._texts_length += 1
-    return self._texts_length
-
-  @property
-  def file_content(self) -> str:
-    file_content: str
-    if self._xmlns is None:
-      file_content = tostring(self._root, encoding="unicode")
-      file_content = to_html(file_content)
-    else:
-      root = Element(
-        self._root.tag,
-        attrib={**self._root.attrib, "xmlns": self._xmlns},
-      )
-      root.extend(self._root)
-      # XHTML disable <tag/> (we need replace them with <tag></tag>)
-      for element in _all_elements(root):
-        if element.text is None:
-          element.text = ""
-      file_content = tostring(root, encoding="unicode")
-
-    if self._head is not None:
-      file_content = self._head + file_content
-    return file_content
-
-def _all_elements(parent: Element):
-  yield parent
-  for child in parent:
-    yield from _all_elements(child)

epub_translator/epub/html/texts_searcher.py

@@ -1,46 +0,0 @@
-from typing import Generator, TypeGuard
-from enum import auto, Enum
-from xml.etree.ElementTree import Element
-
-
-class TextPosition(Enum):
-  WHOLE_DOM = auto()
-  TEXT = auto()
-  TAIL = auto()
-
-# element, position, parent
-TextDescription = tuple[Element, TextPosition, Element | None]
-
-_IGNORE_TAGS = (
-  "title", "link", "style", "css", "img", "script", "metadata",
-  "{http://www.w3.org/1998/Math/MathML}math", # TODO: 公式是正文，也要读进去，暂时忽略避免扰乱得了。
-)
-
-_TEXT_LEAF_TAGS = (
-  "a", "b", "br", "hr", "span", "em", "strong", "label",
-)
-
-def search_texts(element: Element, parent: Element | None = None) -> Generator[TextDescription, None, None]:
-  if element.tag in _IGNORE_TAGS:
-    return
-
-  if any(c.tag not in _TEXT_LEAF_TAGS for c in element):
-    if _is_not_empty_str(element.text):
-      yield element, TextPosition.TEXT, parent
-    for child in element:
-      if child.tag in _TEXT_LEAF_TAGS:
-        yield child, TextPosition.WHOLE_DOM, element
-      else:
-        yield from search_texts(child, element)
-      if _is_not_empty_str(child.tail):
-        yield child, TextPosition.TAIL, element
-  else:
-    yield element, TextPosition.WHOLE_DOM, parent
-
-def _is_not_empty_str(text: str | None) -> TypeGuard[str]:
-  if text is None:
-    return False
-  for char in text:
-    if char not in (" ", "\n"):
-      return True
-  return False

epub_translator/llm/node.py

@@ -1,201 +0,0 @@
-import datetime
-
-from os import PathLike
-from pathlib import Path
-from typing import cast, Any, TypeVar, Generator, Sequence, Callable
-from importlib.resources import files
-from jinja2 import Environment, Template
-from xml.etree.ElementTree import Element
-from pydantic import SecretStr
-from logging import getLogger, DEBUG, Formatter, Logger, FileHandler
-from tiktoken import get_encoding, Encoding
-from langchain_core.messages import SystemMessage, HumanMessage
-
-from ..template import create_env
-from ..xml import decode_friendly, encode_friendly
-from .increasable import Increasable
-from .executor import LLMExecutor
-
-
-R = TypeVar("R")
-
-class LLM:
-  def __init__(
-      self,
-      key: str,
-      url: str,
-      model: str,
-      token_encoding: str,
-      timeout: float | None = None,
-      top_p: float | tuple[float, float] | None = None,
-      temperature: float | tuple[float, float] | None = None,
-      retry_times: int = 5,
-      retry_interval_seconds: float = 6.0,
-      log_dir_path: PathLike | None = None,
-    ):
-    prompts_path = files("epub_translator") / "data"
-    self._templates: dict[str, Template] = {}
-    self._encoding: Encoding = get_encoding(token_encoding)
-    self._env: Environment = create_env(prompts_path)
-    self._logger_save_path: Path | None = None
-
-    if log_dir_path is not None:
-      self._logger_save_path = Path(log_dir_path)
-      if not self._logger_save_path.exists():
-        self._logger_save_path.mkdir(parents=True, exist_ok=True)
-      elif not self._logger_save_path.is_dir():
-        self._logger_save_path = None
-
-    self._executor = LLMExecutor(
-      url=url,
-      model=model,
-      api_key=cast(SecretStr, key),
-      timeout=timeout,
-      top_p=Increasable(top_p),
-      temperature=Increasable(temperature),
-      retry_times=retry_times,
-      retry_interval_seconds=retry_interval_seconds,
-      create_logger=self._create_logger,
-    )
-
-  def _create_logger(self) -> Logger | None:
-    if self._logger_save_path is None:
-      return None
-
-    now = datetime.datetime.now(datetime.timezone.utc)
-    timestamp = now.strftime("%Y-%m-%d %H-%M-%S %f")
-    file_path = self._logger_save_path / f"request {timestamp}.log"
-    logger = getLogger(f"LLM Request {timestamp}")
-    logger.setLevel(DEBUG)
-    handler = FileHandler(file_path, encoding="utf-8")
-    handler.setLevel(DEBUG)
-    handler.setFormatter(Formatter("%(asctime)s    %(message)s", "%H:%M:%S"))
-    logger.addHandler(handler)
-
-    return logger
-
-  def request_text(
-      self,
-      template_name: str,
-      text_tag: str,
-      user_data: Element | str,
-      parser: Callable[[str], R],
-      max_tokens: int | None = None,
-      params: dict[str, Any] | None = None,
-    ) -> R:
-
-    if params is None:
-      params = {}
-
-    def parse_response(response: str) -> R:
-      text = next(self._search_quotes(text_tag.lower(), response), None)
-      if text is None:
-        raise ValueError(f"No valid {text_tag} response found")
-      return parser(text)
-
-    return self._executor.request(
-      input=self._create_input(template_name, user_data, params),
-      parser=parse_response,
-      max_tokens=max_tokens,
-    )
-
-  def request_xml(
-        self,
-        template_name: str,
-        user_data: Element | str,
-        parser: Callable[[Element], R],
-        max_tokens: int | None = None,
-        params: dict[str, Any] | None = None,
-      ) -> R:
-
-    if params is None:
-      params = {}
-
-    def parse_response(response: str) -> R:
-      element = next(decode_friendly(response, "response"), None)
-      if element is None:
-        raise ValueError("No valid XML response found")
-      return parser(element)
-
-    return self._executor.request(
-      input=self._create_input(template_name, user_data, params),
-      parser=parse_response,
-      max_tokens=max_tokens,
-    )
-
-  def _create_input(self, template_name: str, user_data: Element | str, params: dict[str, Any]):
-    data: str
-    if isinstance(user_data, Element):
-      data = encode_friendly(user_data)
-      data = f"```XML\n{data}\n```"
-    else:
-      data = user_data
-
-    template = self._template(template_name)
-    prompt = template.render(**params)
-    return [
-      SystemMessage(content=prompt),
-      HumanMessage(content=data)
-    ]
-
-  def prompt_tokens_count(self, template_name: str, params: dict[str, Any]) -> int:
-    template = self._template(template_name)
-    prompt = template.render(**params)
-    return len(self._encoding.encode(prompt))
-
-  def encode_tokens(self, text: str) -> list[int]:
-    return self._encoding.encode(text)
-
-  def decode_tokens(self, tokens: Sequence[int]) -> str:
-    return self._encoding.decode(tokens)
-
-  def count_tokens_count(self, text: str) -> int:
-    return len(self._encoding.encode(text))
-
-  def _template(self, template_name: str) -> Template:
-    template = self._templates.get(template_name, None)
-    if template is None:
-      template = self._env.get_template(template_name)
-      self._templates[template_name] = template
-    return template
-
-  def _search_quotes(self, kind: str, response: str) -> Generator[str, None, None]:
-    start_marker = f"```{kind}"
-    end_marker = "```"
-    start_index = 0
-
-    while True:
-      start_index = self._find_ignore_case(
-        raw=response,
-        sub=start_marker,
-        start=start_index,
-      )
-      if start_index == -1:
-        break
-
-      end_index = self._find_ignore_case(
-        raw=response,
-        sub=end_marker,
-        start=start_index + len(start_marker),
-      )
-      if end_index == -1:
-        break
-
-      extracted_text = response[start_index + len(start_marker):end_index].strip()
-      yield extracted_text
-      start_index = end_index + len(end_marker)
-
-  def _find_ignore_case(self, raw: str, sub: str, start: int = 0):
-    if not sub:
-      return 0 if 0 >= start else -1
-
-    raw_len, sub_len = len(raw), len(sub)
-    for i in range(start, raw_len - sub_len + 1):
-      match = True
-      for j in range(sub_len):
-        if raw[i + j].lower() != sub[j].lower():
-          match = False
-          break
-      if match:
-        return i
-    return -1

epub_translator/translation/__init__.py

@@ -1,2 +0,0 @@
-from .types import Incision, Fragment, Language
-from .translation import translate, ProgressReporter

epub_translator/translation/chunk.py

@@ -1,118 +0,0 @@
-from dataclasses import dataclass
-from typing import Iterator, Iterable, Generator
-from hashlib import sha512
-from ..llm import LLM
-from .types import Fragment, Language
-
-
-@dataclass
-class Chunk:
-  index: int
-  hash: bytes
-  head: list[str]
-  body: list[str]
-  tail: list[str]
-  tokens_count: int
-
-@dataclass
-class ChunkRange:
-  index: int
-  head_remain_tokens: int
-  tail_remain_tokens: int
-  head_index: int
-  body_index: int
-  tail_index: int
-  fragments_count: int
-  tokens_count: int
-
-  def match(self, index: int) -> bool:
-    return self.head_index <= index < self.head_index + self.fragments_count
-
-def match_fragments(
-        llm: LLM,
-        target_language: Language,
-        chunk_ranges_iter: Iterator[ChunkRange],
-        fragments_iter: Iterator[Fragment],
-      ) -> Generator[Chunk, None, None]:
-
-  for range, texts in _match_range_and_texts(
-    chunk_range_iter=chunk_ranges_iter,
-    fragments_iter=fragments_iter,
-  ):
-    head_length = range.body_index - range.head_index
-    body_length = range.tail_index - range.body_index
-    head = texts[:head_length]
-    body = texts[head_length:head_length + body_length]
-    tail = texts[head_length + body_length:]
-
-    hash = _hash_texts_list(target_language, (head, body, tail))
-    head = _crop_extra_texts(llm, head, True, range.head_remain_tokens)
-    tail = _crop_extra_texts(llm, tail, False, range.tail_remain_tokens)
-
-    yield Chunk(
-      hash=hash,
-      head=head,
-      body=body,
-      tail=tail,
-      index=range.index,
-      tokens_count=range.tokens_count,
-    )
-
-def _match_range_and_texts(
-      chunk_range_iter: Iterator[ChunkRange],
-      fragments_iter: Iterator[Fragment],
-    ) -> Generator[tuple[ChunkRange, list[str]], None, None]:
-
-  next_chunk_range: ChunkRange | None = None
-  matched_chunk_ranges: list[tuple[ChunkRange, list[str]]] = []
-
-  for index, fragment in enumerate(fragments_iter):
-    while True:
-      if next_chunk_range is None:
-        next_chunk_range = next(chunk_range_iter, None)
-        if next_chunk_range is None:
-          break
-      if not next_chunk_range.match(index):
-        break
-      matched_chunk_ranges.append((next_chunk_range, []))
-      next_chunk_range = None
-
-    if matched_chunk_ranges:
-      next_matched_chunks: list[tuple[ChunkRange, list[str]]] = []
-      for chunk_range, texts in matched_chunk_ranges:
-        if chunk_range.match(index):
-          texts.append(fragment.text)
-          next_matched_chunks.append((chunk_range, texts))
-        else:
-          yield chunk_range, texts
-      matched_chunk_ranges = next_matched_chunks
-
-  yield from matched_chunk_ranges
-
-def _hash_texts_list(target_language: Language, texts_iterable: Iterable[list[str]]) -> bytes:
-  m = sha512()
-  m.update(target_language.value.encode("utf-8"))
-  for texts in texts_iterable:
-    for text in texts:
-      m.update(b"\x00")
-      m.update(text.encode("utf-8"))
-  return m.digest()
-
-def _crop_extra_texts(llm: LLM, texts: list[str], crop_left: bool, remain_tokens_count: int):
-  tokens_list: list[list[int]] = [llm.encode_tokens(text) for text in texts]
-  remain_texts: list[str] = []
-
-  for tokens in (reversed(tokens_list) if crop_left else tokens_list):
-    tokens_count = len(tokens)
-    if remain_tokens_count >= tokens_count:
-      remain_tokens_count -= tokens_count
-      remain_texts.append(llm.decode_tokens(tokens))
-      if remain_tokens_count == 0:
-        break
-    else:
-      remain_tokens = tokens[-remain_tokens_count:] if crop_left else tokens[:remain_tokens_count]
-      remain_texts.append(llm.decode_tokens(remain_tokens))
-
-  if crop_left:
-    remain_texts.reverse()
-  return remain_texts

epub_translator/translation/splitter.py

@@ -1,78 +0,0 @@
-from typing import Iterator, Generator
-from resource_segmentation import split, Resource, Segment
-
-from ..llm import LLM
-from .types import Fragment, Incision
-from .chunk import ChunkRange
-
-
-def split_into_chunks(llm: LLM, fragments_iter: Iterator[Fragment], max_chunk_tokens_count: int):
-  for index, group in enumerate(split(
-    resources=_gen_resources(llm, fragments_iter),
-    max_segment_count=max_chunk_tokens_count,
-    gap_rate=0.15,
-    tail_rate=0.5,
-    border_incision=Incision.IMPOSSIBLE,
-  )):
-    head_index: int
-    tail_index: int
-    fragments_count: int
-    body_index, body_end_index, body_tokens_count = _group_part(group.body)
-
-    if group.head:
-      head_index, head_end_index, _ = _group_part(group.head)
-      assert head_end_index + 1 == body_index, "Head must be continuous with body"
-    else:
-      head_index = body_index
-
-    if group.tail:
-      tail_index, tail_end_index, _ = _group_part(group.tail)
-      fragments_count = tail_end_index - head_index + 1
-      assert body_end_index + 1 == tail_index, "Body must be continuous with tail"
-    else:
-      tail_index = body_end_index + 1
-      fragments_count = tail_index - head_index
-
-    yield ChunkRange(
-      index=index,
-      head_remain_tokens=group.head_remain_count,
-      tail_remain_tokens=group.tail_remain_count,
-      head_index=head_index,
-      body_index=body_index,
-      tail_index=tail_index,
-      fragments_count=fragments_count,
-      tokens_count=body_tokens_count,
-    )
-
-def _gen_resources(llm: LLM, fragments_iter: Iterator[Fragment]) -> Generator[Resource[int], None, None]:
-  for index, fragment in enumerate(fragments_iter):
-    yield Resource(
-      count=llm.count_tokens_count(fragment.text),
-      start_incision=fragment.start_incision,
-      end_incision=fragment.end_incision,
-      payload=index,
-    )
-
-def _group_part(target: list[Resource[int] | Segment[int]]) -> tuple[int, int, int]:
-  start_index: int | None = None
-  previous_index: int = 0
-  tokens_count: int = 0
-  for resource in _iter_group_part(target):
-    index = resource.payload
-    if start_index is None:
-      start_index = index
-    else:
-      assert index == previous_index + 1, "Resources in group part must be continuous"
-    previous_index = index
-    tokens_count += resource.count
-
-  assert start_index is not None, "Group part must contain at least one resource"
-  return start_index, previous_index, tokens_count
-
-def _iter_group_part(target: list[Resource[int] | Segment[int]]) -> Generator[Resource[int], None, None]:
-  for item in target:
-    if isinstance(item, Resource):
-      yield item
-    elif isinstance(item, Segment):
-      for resource in item.resources:
-        yield resource

epub_translator/translation/store.py

@@ -1,36 +0,0 @@
-from shutil import rmtree
-from pathlib import Path
-from typing import Iterable
-
-
-class Store:
-  def __init__(self, directory: Path):
-    self._directory = directory
-
-  def get(self, chunk_hash: bytes) -> list[str] | None:
-    file_path = self._file_path(chunk_hash)
-    if not file_path.exists() or not file_path.is_file():
-      return None
-    with file_path.open("r", encoding="utf-8") as file:
-      return file.read().split("\n")
-
-  def put(self, chunk_hash: bytes, lines: Iterable[str]):
-    file_path = self._file_path(chunk_hash)
-    if file_path.exists():
-      if file_path.is_file():
-        file_path.unlink()
-      else:
-        rmtree(file_path)
-
-    file_path.parent.mkdir(parents=True, exist_ok=True)
-    with file_path.open("w", encoding="utf-8") as file:
-      is_first_line = True
-      for line in lines:
-        if is_first_line:
-          is_first_line = False
-        else:
-          file.write("\n")
-        file.write(line)
-
-  def _file_path(self, chunk_hash: bytes) -> Path:
-    return self._directory / f"{chunk_hash.hex()}.chunk"