epub-translator 0.1.5__py3-none-any.whl → 0.1.7__py3-none-any.whl

epub_translator/__init__.py

@@ -1,6 +1,5 @@
-from . import language
 from .llm import LLM
-from .translator import FillFailedEvent, translate
+from .translation import FillFailedEvent, language, translate
 from .xml_translator import SubmitKind
 
 __all__ = [

epub_translator/data/translate.jinja

@@ -13,6 +13,9 @@ Translation rules:
 {% if user_prompt -%}
 User may provide additional requirements in <rules> tags before the source text. Follow them, but prioritize the rules above if conflicts arise.
 
+<rules>
+{{ user_prompt }}
+</rules>
 {% endif -%}
 
 Output only the translated text, nothing else.

epub_translator/epub/__init__.py

@@ -1,4 +1,4 @@
 from .metadata import read_metadata, write_metadata
 from .spines import search_spine_paths
-from .toc import read_toc, write_toc
+from .toc import Toc, read_toc, write_toc
 from .zip import Zip

epub_translator/llm/context.py

@@ -1,5 +1,6 @@
 import hashlib
 import json
+import threading
 import uuid
 from pathlib import Path
 from typing import Self
@@ -8,6 +9,9 @@ from .executor import LLMExecutor
 from .increasable import Increasable, Increaser
 from .types import Message, MessageRole
 
+# Global lock for cache file commit operations
+_CACHE_COMMIT_LOCK = threading.Lock()
+
 
 class LLMContext:
     def __init__(
@@ -101,7 +105,12 @@ class LLMContext:
                 # Remove the .[context-id].txt suffix to get permanent name
                 permanent_name = temp_file.name.rsplit(".", 2)[0] + ".txt"
                 permanent_file = temp_file.parent / permanent_name
-                temp_file.rename(permanent_file)
+
+                with _CACHE_COMMIT_LOCK:  # 多线程下的线程安全
+                    if permanent_file.exists():
+                        temp_file.unlink()
+                    else:
+                        temp_file.rename(permanent_file)
 
     def _rollback(self) -> None:
         for temp_file in self._temp_files:

epub_translator/llm/core.py

@@ -1,4 +1,5 @@
 import datetime
+import threading
 from collections.abc import Generator
 from importlib.resources import files
 from logging import DEBUG, FileHandler, Formatter, Logger, getLogger
@@ -14,6 +15,11 @@ from .executor import LLMExecutor
 from .increasable import Increasable
 from .types import Message
 
+# Global state for logger filename generation
+_LOGGER_LOCK = threading.Lock()
+_LAST_TIMESTAMP: str | None = None
+_LOGGER_SUFFIX_ID: int = 1
+
 
 class LLM:
     def __init__(
@@ -95,13 +101,34 @@ class LLM:
         return dir_path.resolve()
 
     def _create_logger(self) -> Logger | None:
+        # pylint: disable=global-statement
+        global _LAST_TIMESTAMP, _LOGGER_SUFFIX_ID
+
         if self._logger_save_path is None:
             return None
 
         now = datetime.datetime.now(datetime.UTC)
-        timestamp = now.strftime("%Y-%m-%d %H-%M-%S %f")
-        file_path = self._logger_save_path / f"request {timestamp}.log"
-        logger = getLogger(f"LLM Request {timestamp}")
+        # Use second-level precision for collision detection
+        timestamp_key = now.strftime("%Y-%m-%d %H-%M-%S")
+
+        with _LOGGER_LOCK:
+            if _LAST_TIMESTAMP == timestamp_key:
+                _LOGGER_SUFFIX_ID += 1
+                suffix_id = _LOGGER_SUFFIX_ID
+            else:
+                _LAST_TIMESTAMP = timestamp_key
+                _LOGGER_SUFFIX_ID = 1
+                suffix_id = 1
+
+        if suffix_id == 1:
+            file_name = f"request {timestamp_key}.log"
+            logger_name = f"LLM Request {timestamp_key}"
+        else:
+            file_name = f"request {timestamp_key}_{suffix_id}.log"
+            logger_name = f"LLM Request {timestamp_key}_{suffix_id}"
+
+        file_path = self._logger_save_path / file_name
+        logger = getLogger(logger_name)
         logger.setLevel(DEBUG)
         handler = FileHandler(file_path, encoding="utf-8")
         handler.setLevel(DEBUG)

epub_translator/segment/__init__.py

@@ -21,6 +21,7 @@ from .text_segment import (
     TextPosition,
     TextSegment,
     combine_text_segments,
+    find_block_depth,
     incision_between,
     search_text_segments,
 )

epub_translator/segment/inline_segment.py

@@ -47,6 +47,7 @@ def search_inline_segments(text_segments: Iterable[TextSegment]) -> Generator["I
                 inline_segment = _pop_stack_data(stack_data)
                 stack_data = None
                 if inline_segment:
+                    inline_segment.id = 0
                     yield inline_segment
 
         if stack_data is None:
@@ -73,6 +74,7 @@ def search_inline_segments(text_segments: Iterable[TextSegment]) -> Generator["I
     if stack_data is not None:
         inline_segment = _pop_stack_data(stack_data)
         if inline_segment:
+            inline_segment.id = 0
             yield inline_segment
 
 
@@ -115,7 +117,7 @@ class InlineSegment:
         self._child_tag2ids: dict[str, list[int]] = {}
         self._child_tag2count: dict[str, int] = {}
 
-        next_temp_id: int = 0
+        next_temp_id: int = 1
         terms = nest((child.parent.tag, child) for child in children if isinstance(child, InlineSegment))
 
         for tag, child_terms in terms.items():
@@ -162,6 +164,14 @@ class InlineSegment:
             elif isinstance(child, InlineSegment):
                 yield from child
 
+    def clone(self) -> "InlineSegment":
+        cloned_segment = InlineSegment(
+            depth=len(self._parent_stack),
+            children=[child.clone() for child in self._children],
+        )
+        cloned_segment.id = self.id
+        return cloned_segment
+
     def recreate_ids(self, id_generator: IDGenerator) -> None:
         self._child_tag2count.clear()
         self._child_tag2ids.clear()

epub_translator/segment/text_segment.py

@@ -4,7 +4,7 @@ from enum import Enum, auto
 from typing import Self
 from xml.etree.ElementTree import Element
 
-from ..xml import expand_left_element_texts, expand_right_element_texts, is_inline_tag, normalize_text_in_element
+from ..xml import expand_left_element_texts, expand_right_element_texts, is_inline_element, normalize_text_in_element
 
 
 class TextPosition(Enum):
@@ -33,10 +33,6 @@ class TextSegment:
     def block_parent(self) -> Element:
         return self.parent_stack[self.block_depth - 1]
 
-    @property
-    def xml_text(self) -> str:
-        return "".join(_expand_xml_texts(self))
-
     def strip_block_parents(self) -> Self:
         self.parent_stack = self.parent_stack[self.block_depth - 1 :]
         self.block_depth = 1
@@ -104,7 +100,7 @@ def search_text_segments(root: Element) -> Generator[TextSegment, None, None]:
 def _search_text_segments(stack: list[Element], element: Element) -> Generator[TextSegment, None, None]:
     text = normalize_text_in_element(element.text)
     next_stack = stack + [element]
-    next_block_depth = _find_block_depth(next_stack)
+    next_block_depth = find_block_depth(next_stack)
 
     if text is not None:
         yield TextSegment(
@@ -129,12 +125,11 @@ def _search_text_segments(stack: list[Element], element: Element) -> Generator[T
             )
 
 
-def _find_block_depth(parent_stack: list[Element]) -> int:
+def find_block_depth(parent_stack: list[Element]) -> int:
     index: int = 0
-    for i in range(len(parent_stack) - 1, -1, -1):
-        if not is_inline_tag(parent_stack[i].tag):
+    for i in range(len(parent_stack)):
+        if not is_inline_element(parent_stack[i]):
             index = i
-            break
     return index + 1  # depth is a count not index
 
 

epub_translator/segment/utils.py

@@ -8,22 +8,6 @@ def element_fingerprint(element: Element) -> str:
     return f"<{element.tag} {' '.join(attrs)}/>"
 
 
-def unwrap_parents(element: Element) -> tuple[Element, list[Element]]:
-    parents: list[Element] = []
-    while True:
-        if len(element) != 1:
-            break
-        child = element[0]
-        if not element.text:
-            break
-        if not child.tail:
-            break
-        parents.append(element)
-        element = child
-        element.tail = None
-    return element, parents
-
-
 def id_in_element(element: Element) -> int | None:
     id_str = element.get(ID_KEY, None)
     if id_str is None:

epub_translator/translation/__init__.py

@@ -0,0 +1,2 @@
+from . import language
+from .translator import FillFailedEvent, translate

epub_translator/{epub_transcode.py → translation/epub_transcode.py}

@@ -6,8 +6,8 @@ EPUB 数据结构与 XML 的编码/解码转换
 
 from xml.etree.ElementTree import Element
 
-from .epub.metadata import MetadataField
-from .epub.toc import Toc
+from ..epub import Toc
+from ..epub.metadata import MetadataField
 
 
 def encode_toc(toc: Toc) -> Element:

epub_translator/{punctuation.py → translation/punctuation.py}

@@ -1,6 +1,6 @@
 from xml.etree.ElementTree import Element
 
-from .xml import iter_with_stack
+from ..xml import iter_with_stack
 
 _QUOTE_MAPPING = {
     # 法语引号

epub_translator/{translator.py → translation/translator.py}

@@ -5,7 +5,7 @@ from importlib.metadata import version as get_package_version
 from os import PathLike
 from pathlib import Path
 
-from .epub import (
+from ..epub import (
     Zip,
     read_metadata,
     read_toc,
@@ -13,12 +13,12 @@ from .epub import (
     write_metadata,
     write_toc,
 )
+from ..llm import LLM
+from ..xml import XMLLikeNode, deduplicate_ids_in_element, find_first
+from ..xml_translator import FillFailedEvent, SubmitKind, TranslationTask, XMLTranslator
 from .epub_transcode import decode_metadata, decode_toc_list, encode_metadata, encode_toc_list
-from .llm import LLM
 from .punctuation import unwrap_french_quotes
-from .xml import XMLLikeNode, deduplicate_ids_in_element, find_first
 from .xml_interrupter import XMLInterrupter
-from .xml_translator import FillFailedEvent, SubmitKind, TranslationTask, XMLTranslator
 
 
 class _ElementType(Enum):
@@ -40,7 +40,8 @@ def translate(
     submit: SubmitKind,
     user_prompt: str | None = None,
     max_retries: int = 5,
-    max_group_tokens: int = 1200,
+    max_group_tokens: int = 2600,
+    concurrency: int = 1,
     llm: LLM | None = None,
     translation_llm: LLM | None = None,
     fill_llm: LLM | None = None,
@@ -62,7 +63,7 @@ def translate(
         ignore_translated_error=False,
         max_retries=max_retries,
         max_fill_displaying_errors=10,
-        max_group_tokens=max_group_tokens,
+        max_group_score=max_group_tokens,
         cache_seed_content=f"{_get_version()}:{target_language}",
     )
     with Zip(
@@ -92,6 +93,7 @@ def translate(
         current_progress = 0.0
 
         for translated_elem, context in translator.translate_elements(
+            concurrency=concurrency,
             interrupt_source_text_segments=interrupter.interrupt_source_text_segments,
             interrupt_translated_text_segments=interrupter.interrupt_translated_text_segments,
             interrupt_block_element=interrupter.interrupt_block_element,

epub_translator/{xml_interrupter.py → translation/xml_interrupter.py}

@@ -1,9 +1,13 @@
 from collections.abc import Generator, Iterable
 from typing import cast
-from xml.etree.ElementTree import Element
+from xml.etree.ElementTree import Element, tostring
 
-from .segment import TextSegment
-from .utils import ensure_list, normalize_whitespace
+from bs4 import BeautifulSoup
+from mathml2latex.mathml import process_mathml
+
+from ..segment import TextSegment, combine_text_segments, find_block_depth
+from ..utils import ensure_list
+from ..xml import clone_element
 
 _ID_KEY = "__XML_INTERRUPTER_ID"
 _MATH_TAG = "math"
@@ -37,8 +41,10 @@ class XMLInterrupter:
     def interrupt_block_element(self, element: Element) -> Element:
         interrupted_element = self._placeholder2interrupted.pop(id(element), None)
         if interrupted_element is None:
+            element.attrib.pop(_ID_KEY, None)
             return element
         else:
+            interrupted_element.attrib.pop(_ID_KEY, None)
             return interrupted_element
 
     def _expand_source_text_segment(self, text_segment: TextSegment):
@@ -81,14 +87,18 @@ class XMLInterrupter:
                     _ID_KEY: cast(str, interrupted_element.get(_ID_KEY)),
                 },
             )
+            interrupted_display = interrupted_element.get("display", None)
+            if interrupted_display is not None:
+                placeholder_element.set("display", interrupted_display)
+
             raw_parent_stack = text_segment.parent_stack[:interrupted_index]
             parent_stack = raw_parent_stack + [placeholder_element]
             merged_text_segment = TextSegment(
-                text="".join(t.text for t in text_segments),
+                text=self._render_latex(text_segments),
                 parent_stack=parent_stack,
                 left_common_depth=text_segments[0].left_common_depth,
                 right_common_depth=text_segments[-1].right_common_depth,
-                block_depth=len(parent_stack),
+                block_depth=find_block_depth(parent_stack),
                 position=text_segments[0].position,
             )
             self._placeholder2interrupted[id(placeholder_element)] = interrupted_element
@@ -116,8 +126,8 @@ class XMLInterrupter:
                 # 原始栈退光，仅留下相对 interrupted 元素的栈，这种格式与 translated 要求一致
                 text_segment.left_common_depth = max(0, text_segment.left_common_depth - interrupted_index)
                 text_segment.right_common_depth = max(0, text_segment.right_common_depth - interrupted_index)
-                text_segment.block_depth = 1
                 text_segment.parent_stack = text_segment.parent_stack[interrupted_index:]
+                text_segment.block_depth = find_block_depth(text_segment.parent_stack)
 
         return merged_text_segment
 
@@ -129,37 +139,51 @@ class XMLInterrupter:
                 break
         return interrupted_index
 
+    def _render_latex(self, text_segments: list[TextSegment]) -> str:
+        math_element, _ = next(combine_text_segments(text_segments))
+        while math_element.tag != _MATH_TAG:
+            if len(math_element) == 0:
+                return ""
+            math_element = math_element[0]
+
+        math_element = clone_element(math_element)
+        math_element.attrib.pop(_ID_KEY, None)
+        math_element.tail = None
+        latex: str | None = None
+        try:
+            mathml_str = tostring(math_element, encoding="unicode")
+            soup = BeautifulSoup(mathml_str, "html.parser")
+            latex = process_mathml(soup)
+        except Exception:
+            pass
+
+        if latex is None:
+            latex = "".join(t.text for t in text_segments)
+        elif math_element.get("display", None) == "inline":
+            latex = f"${latex}$"
+        else:
+            latex = f"$${latex}$$"
+
+        return f" {latex} "
+
     def _expand_translated_text_segment(self, text_segment: TextSegment):
-        interrupted_id = text_segment.block_parent.attrib.pop(_ID_KEY, None)
+        parent_element = text_segment.parent_stack[-1]
+        interrupted_id = parent_element.attrib.pop(_ID_KEY, None)
         if interrupted_id is None:
             yield text_segment
             return
 
-        raw_text_segments = self._raw_text_segments.pop(interrupted_id, None)
-        if not raw_text_segments:
+        if parent_element is text_segment.block_parent:
+            # Block-level math， need to be hidden
             return
 
-        raw_block = raw_text_segments[0].parent_stack[0]
-        if not self._is_inline_math(raw_block):
+        raw_text_segments = self._raw_text_segments.pop(interrupted_id, None)
+        if not raw_text_segments:
+            yield text_segment
             return
 
         for raw_text_segment in raw_text_segments:
+            text_basic_parent_stack = text_segment.parent_stack[:-1]
             raw_text_segment.block_parent.attrib.pop(_ID_KEY, None)
+            raw_text_segment.parent_stack = text_basic_parent_stack + raw_text_segment.parent_stack
             yield raw_text_segment
-
-    def _has_no_math_texts(self, element: Element):
-        if element.tag == _MATH_TAG:
-            return True
-        if element.text and normalize_whitespace(element.text).strip():
-            return False
-        for child_element in element:
-            if not self._has_no_math_texts(child_element):
-                return False
-            if child_element.tail and normalize_whitespace(child_element.tail).strip():
-                return False
-        return True
-
-    def _is_inline_math(self, element: Element) -> bool:
-        if element.tag != _MATH_TAG:
-            return False
-        return element.get("display", "").lower() != "block"

epub_translator/xml/__init__.py

@@ -1,6 +1,6 @@
 from .const import *
 from .deduplication import *
-from .firendly import *
+from .friendly import *
 from .inline import *
 from .utils import *
 from .xml import *

epub_translator/xml/inline.py

@@ -1,6 +1,9 @@
+from xml.etree.ElementTree import Element
+
 # HTML inline-level elements
 # Reference: https://developer.mozilla.org/en-US/docs/Web/HTML/Inline_elements
 # Reference: https://developer.mozilla.org/en-US/docs/Glossary/Inline-level_content
+# Reference: https://developer.mozilla.org/en-US/docs/MathML/Element
 _HTML_INLINE_TAGS = frozenset(
     (
         # Inline text semantics
@@ -59,9 +62,52 @@ _HTML_INLINE_TAGS = frozenset(
         "del",
         "ins",
         "slot",
+        # MathML elements
+        # Token elements
+        "mi",  # identifier
+        "mn",  # number
+        "mo",  # operator
+        "ms",  # string literal
+        "mspace",  # space
+        "mtext",  # text
+        # General layout
+        "menclose",  # enclosed content
+        "merror",  # syntax error message
+        "mfenced",  # parentheses (deprecated)
+        "mfrac",  # fraction
+        "mpadded",  # space around content
+        "mphantom",  # invisible content
+        "mroot",  # radical with index
+        "mrow",  # grouped sub-expressions
+        "msqrt",  # square root
+        "mstyle",  # style change
+        # Scripts and limits
+        "mmultiscripts",  # prescripts and tensor indices
+        "mover",  # overscript
+        "mprescripts",  # prescripts separator
+        "msub",  # subscript
+        "msubsup",  # subscript-superscript pair
+        "msup",  # superscript
+        "munder",  # underscript
+        "munderover",  # underscript-overscript pair
+        # Table math
+        "mtable",  # table or matrix
+        "mtr",  # row in table or matrix
+        "mtd",  # cell in table or matrix
+        # Semantic annotations
+        "annotation",  # data annotation
+        "annotation-xml",  # XML annotation
+        "semantics",  # semantic annotation container
+        # Other
+        "maction",  # bind actions to sub-expressions (deprecated)
     )
 )
 
 
-def is_inline_tag(tag: str) -> bool:
-    return tag.lower() in _HTML_INLINE_TAGS
+def is_inline_element(element: Element) -> bool:
+    if element.tag.lower() in _HTML_INLINE_TAGS:
+        return True
+    display = element.get("display", None)
+    if display is not None and display.lower() == "inline":
+        return True
+    return False

epub_translator/xml_translator/concurrency.py

@@ -0,0 +1,52 @@
+from collections import deque
+from collections.abc import Callable, Iterable
+from concurrent.futures import Future, ThreadPoolExecutor
+from typing import TypeVar
+
+P = TypeVar("P")
+R = TypeVar("R")
+
+
+def run_concurrency(
+    parameters: Iterable[P],
+    execute: Callable[[P], R],
+    concurrency: int,
+) -> Iterable[R]:
+    assert concurrency >= 1, "the concurrency must be at least 1"
+    # Fast path: concurrency == 1, no thread overhead
+    if concurrency == 1:
+        for param in parameters:
+            yield execute(param)
+        return
+
+    executor = ThreadPoolExecutor(max_workers=concurrency)
+    did_shutdown = False
+    try:
+        futures: deque[Future[R]] = deque()
+        params_iter = iter(parameters)
+        for _ in range(concurrency):
+            try:
+                param = next(params_iter)
+                future = executor.submit(execute, param)
+                futures.append(future)
+            except StopIteration:
+                break
+
+        while futures:
+            future = futures.popleft()
+            yield future.result()
+            try:
+                param = next(params_iter)
+                new_future = executor.submit(execute, param)
+                futures.append(new_future)
+            except StopIteration:
+                pass
+
+    except KeyboardInterrupt:
+        executor.shutdown(wait=False, cancel_futures=True)
+        did_shutdown = True
+        raise
+
+    finally:
+        if not did_shutdown:
+            executor.shutdown(wait=True)