epub-translator 0.1.1__py3-none-any.whl → 0.1.4__py3-none-any.whl

epub_translator/xml/self_closing.py

@@ -0,0 +1,182 @@
+import re
+
+# Some non-standard EPUB generators use HTML-style tags without self-closing syntax
+# We need to convert them to XML-compatible format before parsing
+# These are HTML5 void elements that must be self-closing in XHTML
+_VOID_TAGS = (
+    "area",
+    "base",
+    "br",
+    "col",
+    "embed",
+    "hr",
+    "img",
+    "input",
+    "link",
+    "meta",
+    "param",
+    "source",
+    "track",
+    "wbr",
+)
+
+
+def self_close_void_elements(xml_content: str) -> str:
+    """
+    Convert void HTML elements to self-closing format for XML parsing.
+
+    This function handles non-standard HTML where void elements are not self-closed.
+    For illegal cases like <meta>content</meta>, the content is removed.
+
+    Args:
+        xml_content: HTML/XHTML content string
+
+    Returns:
+        Content with void elements in self-closing format
+
+    Example:
+        <meta charset="utf-8"> → <meta charset="utf-8" />
+        <br> → <br />
+        <meta>illegal</meta> → <meta />
+    """
+    for tag in _VOID_TAGS:
+        xml_content = _fix_void_element(xml_content, tag)
+    return xml_content
+
+
+def _fix_void_element(content: str, tag_name: str) -> str:
+    """
+    Fix a specific void element in the content.
+
+    Strategy:
+    1. Find <tag ...> (not already self-closed)
+    2. Check if there's a matching </tag>
+    3. If yes, remove everything between them and make it self-closing
+    4. If no, just make the opening tag self-closing
+    """
+    result = []
+    pos = 0
+
+    while pos < len(content):
+        tag_start = content.find(f"<{tag_name}", pos)
+        if tag_start == -1:
+            result.append(content[pos:])
+            break
+
+        # Verify it's a complete tag match (not a prefix like <br matching <brain>)
+        # The character after tag_name must be >, /, or whitespace
+        check_pos = tag_start + len(f"<{tag_name}")
+        if check_pos < len(content):
+            next_char = content[check_pos]
+            if next_char not in (">", "/", " ", "\t", "\n", "\r"):
+                result.append(content[pos:check_pos])
+                pos = check_pos
+                continue
+
+        result.append(content[pos:tag_start])
+        tag_end = _find_tag_end(content, tag_start)
+        if tag_end == -1:
+            result.append(content[tag_start:])
+            break
+
+        opening_tag = content[tag_start : tag_end + 1]
+
+        if opening_tag.rstrip().endswith("/>"):
+            result.append(opening_tag)
+            pos = tag_end + 1
+            continue
+
+        if not opening_tag.endswith(">"):
+            result.append(opening_tag)
+            pos = tag_end + 1
+            continue
+
+        closing_tag = f"</{tag_name}>"
+        closing_pos = content.find(closing_tag, tag_end + 1)
+
+        if closing_pos != -1:
+            attrs_part = opening_tag[len(f"<{tag_name}") : -1].rstrip()
+            if attrs_part:
+                result.append(f"<{tag_name}{attrs_part} />")
+            else:
+                result.append(f"<{tag_name} />")
+            pos = closing_pos + len(closing_tag)
+        else:
+            attrs_part = opening_tag[len(f"<{tag_name}") : -1].rstrip()
+            if attrs_part:
+                result.append(f"<{tag_name}{attrs_part} />")
+            else:
+                result.append(f"<{tag_name} />")
+            pos = tag_end + 1
+
+    return "".join(result)
+
+
+def _find_tag_end(content: str, start_pos: int) -> int:
+    """
+    Find the end of an HTML tag (the position of >).
+
+    Handles quotes: ignores > inside quoted attribute values.
+    """
+    pos = start_pos
+    in_quote = None  # None, '"', or "'"
+
+    while pos < len(content):
+        char = content[pos]
+
+        if in_quote:
+            if char == in_quote:
+                if pos > 0 and content[pos - 1] == "\\":
+                    pos += 1
+                    continue
+                else:
+                    in_quote = None
+        else:
+            if char in ('"', "'"):
+                in_quote = char
+            elif char == ">":
+                return pos
+
+        pos += 1
+
+    return -1  # Not found
+
+
+# For saving: match self-closing tags like <br /> or <br/>
+# Capture tag name and everything between tag name and />
+_VOID_TAG_CLOSE_PATTERN = re.compile(r"<(" + "|".join(_VOID_TAGS) + r")([^>]*?)\s*/>")
+
+
+def unclose_void_elements(xml_content: str) -> str:
+    """
+    Convert void elements from self-closing to unclosed format for HTML compatibility.
+
+    Transforms self-closed void elements like <br /> back to <br> for
+    compatibility with HTML parsers that don't support XHTML syntax.
+    Used only for text/html media type files.
+
+    Args:
+        xml_content: HTML/XHTML content string
+
+    Returns:
+        Content with void elements in unclosed format
+
+    Example:
+        <meta charset="utf-8" /> → <meta charset="utf-8">
+        <br /> → <br>
+        <img src="test.png" /> → <img src="test.png">
+    """
+
+    def replacer(m: re.Match):
+        tag_name = m.group(1)
+        attrs = m.group(2).rstrip()  # Remove trailing whitespace
+        if attrs:
+            return f"<{tag_name}{attrs}>"
+        else:
+            return f"<{tag_name}>"
+
+    return re.sub(
+        pattern=_VOID_TAG_CLOSE_PATTERN,
+        repl=replacer,
+        string=xml_content,
+    )

epub_translator/xml/utils.py

@@ -0,0 +1,42 @@
+from collections.abc import Generator
+from xml.etree.ElementTree import Element
+
+from ..utils import normalize_whitespace
+from .const import ID_KEY
+
+
+def normalize_text_in_element(text: str | None) -> str | None:
+    if text is None:
+        return None
+    text = normalize_whitespace(text)
+    if not text.strip():
+        return None
+    return text
+
+
+def append_text_in_element(origin_text: str | None, append_text: str) -> str:
+    if origin_text is None:
+        return append_text
+    else:
+        return origin_text + append_text
+
+
+def index_of_parent(parent: Element, checked_element: Element) -> int:
+    for i, child in enumerate(parent):
+        if child == checked_element:
+            return i
+    raise ValueError("Element not found in parent.")
+
+
+def expand_left_element_texts(element: Element) -> Generator[str, None, None]:
+    yield "<"
+    yield element.tag
+    yield " "
+    yield ID_KEY
+    yield '="99">'
+
+
+def expand_right_element_texts(element: Element) -> Generator[str, None, None]:
+    yield "</"
+    yield element.tag
+    yield ">"

epub_translator/xml/xml.py

@@ -12,6 +12,13 @@ def find_first(element: Element, tag: str) -> Element | None:
     return None
 
 
+def index_in_parent(parent: Element, element: Element) -> int | None:
+    for i, child in enumerate(parent):
+        if child is element:
+            return i
+    return None
+
+
 def iter_with_stack(element: Element) -> Generator[tuple[list[Element], Element], None, None]:
     """先序遍历：yield parent_path, element"""
     stack: list[list[Element]] = [[element]]

epub_translator/xml/xml_like.py

@@ -4,6 +4,7 @@ import warnings
 from typing import IO
 from xml.etree.ElementTree import Element, fromstring, tostring
 
+from .self_closing import self_close_void_elements, unclose_void_elements
 from .xml import iter_with_stack
 
 _XML_NAMESPACE_URI = "http://www.w3.org/XML/1998/namespace"
@@ -31,28 +32,11 @@ _ENCODING_PATTERN = re.compile(r'encoding\s*=\s*["\']([^"\']+)["\']', re.IGNOREC
 _FIRST_ELEMENT_PATTERN = re.compile(r"<(?![?!])[a-zA-Z]")
 _NAMESPACE_IN_TAG = re.compile(r"\{([^}]+)\}")
 
-# Some non-standard EPUB generators use HTML-style tags without self-closing syntax
-# We need to convert them to XML-compatible format before parsing
-_EMPTY_TAGS = (
-    "br",
-    "hr",
-    "input",
-    "col",
-    "base",
-    "meta",
-    "area",
-)
-
-# For reading: match tags like <br> or <br class="x"> (but not <br/> or <body>)
-_EMPTY_TAG_OPEN_PATTERN = re.compile(r"<(" + "|".join(_EMPTY_TAGS) + r")(\s[^/>]*)>")
-
-# For saving: match self-closing tags like <br />
-_EMPTY_TAG_CLOSE_PATTERN = re.compile(r"<(" + "|".join(_EMPTY_TAGS) + r")(\s[^>]*?)\s*/>")
-
 
 class XMLLikeNode:
     def __init__(self, file: IO[bytes], is_html_like: bool = False) -> None:
         raw_content = file.read()
+        self._is_html_like = is_html_like
         self._encoding: str = self._detect_encoding(raw_content)
         content = raw_content.decode(self._encoding)
         self._header, xml_content = self._extract_header(content)
@@ -60,16 +44,9 @@ class XMLLikeNode:
         self._tag_to_namespace: dict[str, str] = {}
         self._attr_to_namespace: dict[str, str] = {}
 
-        # For non-standard HTML files, convert <br> to <br/> before parsing
-        self._is_html_like = is_html_like
-        if is_html_like:
-            xml_content = re.sub(
-                pattern=_EMPTY_TAG_OPEN_PATTERN,
-                repl=lambda m: f"<{m.group(1)}{m.group(2)} />",
-                string=xml_content,
-            )
-
         try:
+            # 不必判断类型，这是一个防御性极强的函数，可做到 shit >> XML
+            xml_content = self_close_void_elements(xml_content)
             self.element = self._extract_and_clean_namespaces(
                 element=fromstring(xml_content),
             )
@@ -92,13 +69,11 @@ class XMLLikeNode:
 
             content = self._serialize_with_namespaces(self.element)
 
-            # For non-standard HTML files, convert back from <br/> to <br>
+            # For non-standard HTML files (text/html), convert back from <br/> to <br>
+            # to maintain compatibility with HTML parsers that don't support XHTML
+            # For XHTML files (application/xhtml+xml), keep self-closing format
             if self._is_html_like:
-                content = re.sub(
-                    pattern=_EMPTY_TAG_CLOSE_PATTERN,
-                    repl=lambda m: f"<{m.group(1)}{m.group(2)}>",
-                    string=content,
-                )
+                content = unclose_void_elements(content)
 
             writer.write(content)
 

epub_translator/xml_interrupter.py

@@ -0,0 +1,165 @@
+from collections.abc import Generator, Iterable
+from typing import cast
+from xml.etree.ElementTree import Element
+
+from .segment import TextSegment
+from .utils import ensure_list, normalize_whitespace
+
+_ID_KEY = "__XML_INTERRUPTER_ID"
+_MATH_TAG = "math"
+_EXPRESSION_TAG = "expression"
+
+
+class XMLInterrupter:
+    def __init__(self) -> None:
+        self._next_id: int = 1
+        self._last_interrupted_id: str | None = None
+        self._placeholder2interrupted: dict[int, Element] = {}
+        self._raw_text_segments: dict[str, list[TextSegment]] = {}
+
+    def interrupt_source_text_segments(
+        self, text_segments: Iterable[TextSegment]
+    ) -> Generator[TextSegment, None, None]:
+        for text_segment in text_segments:
+            yield from self._expand_source_text_segment(text_segment)
+
+        if self._last_interrupted_id is not None:
+            merged_text_segment = self._pop_and_merge_from_buffered(self._last_interrupted_id)
+            if merged_text_segment:
+                yield merged_text_segment
+
+    def interrupt_translated_text_segments(
+        self, text_segments: Iterable[TextSegment]
+    ) -> Generator[TextSegment, None, None]:
+        for text_segment in text_segments:
+            yield from self._expand_translated_text_segment(text_segment)
+
+    def interrupt_block_element(self, element: Element) -> Element:
+        interrupted_element = self._placeholder2interrupted.pop(id(element), None)
+        if interrupted_element is None:
+            return element
+        else:
+            return interrupted_element
+
+    def _expand_source_text_segment(self, text_segment: TextSegment):
+        interrupted_index = self._interrupted_index(text_segment)
+        interrupted_id: str | None = None
+
+        if interrupted_index is not None:
+            interrupted_element = text_segment.parent_stack[interrupted_index]
+            interrupted_id = interrupted_element.get(_ID_KEY)
+            if interrupted_id is None:
+                interrupted_id = str(self._next_id)
+                interrupted_element.set(_ID_KEY, interrupted_id)
+                self._next_id += 1
+            text_segments = ensure_list(
+                target=self._raw_text_segments,
+                key=interrupted_id,
+            )
+            text_segments.append(text_segment)
+
+        if self._last_interrupted_id is not None and interrupted_id != self._last_interrupted_id:
+            merged_text_segment = self._pop_and_merge_from_buffered(self._last_interrupted_id)
+            if merged_text_segment:
+                yield merged_text_segment
+
+        self._last_interrupted_id = interrupted_id
+
+        if interrupted_index is None:
+            yield text_segment
+
+    def _pop_and_merge_from_buffered(self, interrupted_id: str) -> TextSegment | None:
+        merged_text_segment: TextSegment | None = None
+        text_segments = self._raw_text_segments.get(interrupted_id, None)
+        if text_segments:
+            text_segment = text_segments[0]
+            interrupted_index = cast(int, self._interrupted_index(text_segment))
+            interrupted_element = text_segment.parent_stack[cast(int, interrupted_index)]
+            placeholder_element = Element(
+                _EXPRESSION_TAG,
+                {
+                    _ID_KEY: cast(str, interrupted_element.get(_ID_KEY)),
+                },
+            )
+            raw_parent_stack = text_segment.parent_stack[:interrupted_index]
+            parent_stack = raw_parent_stack + [placeholder_element]
+            merged_text_segment = TextSegment(
+                text="".join(t.text for t in text_segments),
+                parent_stack=parent_stack,
+                left_common_depth=text_segments[0].left_common_depth,
+                right_common_depth=text_segments[-1].right_common_depth,
+                block_depth=len(parent_stack),
+                position=text_segments[0].position,
+            )
+            self._placeholder2interrupted[id(placeholder_element)] = interrupted_element
+
+            # TODO: 比较难搞，先关了再说
+            # parent_element: Element | None = None
+            # if interrupted_index > 0:
+            #     parent_element = text_segment.parent_stack[interrupted_index - 1]
+
+            # if (
+            #     not self._is_inline_math(interrupted_element)
+            #     or parent_element is None
+            #     or self._has_no_math_texts(parent_element)
+            # ):
+            #     # 区块级公式不必重复出现，出现时突兀。但行内公式穿插在译文中更有利于读者阅读顺畅。
+            #     self._raw_text_segments.pop(interrupted_id, None)
+            # else:
+            #     for text_segment in text_segments:
+            #         # 原始栈退光，仅留下相对 interrupted 元素的栈，这种格式与 translated 要求一致
+            #         text_segment.left_common_depth = max(0, text_segment.left_common_depth - interrupted_index)
+            #         text_segment.right_common_depth = max(0, text_segment.right_common_depth - interrupted_index)
+            #         text_segment.block_depth = 1
+            #         text_segment.parent_stack = text_segment.parent_stack[interrupted_index:]
+            for text_segment in text_segments:
+                # 原始栈退光，仅留下相对 interrupted 元素的栈，这种格式与 translated 要求一致
+                text_segment.left_common_depth = max(0, text_segment.left_common_depth - interrupted_index)
+                text_segment.right_common_depth = max(0, text_segment.right_common_depth - interrupted_index)
+                text_segment.block_depth = 1
+                text_segment.parent_stack = text_segment.parent_stack[interrupted_index:]
+
+        return merged_text_segment
+
+    def _interrupted_index(self, text_segment: TextSegment) -> int | None:
+        interrupted_index: int | None = None
+        for i, parent_element in enumerate(text_segment.parent_stack):
+            if parent_element.tag == _MATH_TAG:
+                interrupted_index = i
+                break
+        return interrupted_index
+
+    def _expand_translated_text_segment(self, text_segment: TextSegment):
+        interrupted_id = text_segment.block_parent.attrib.pop(_ID_KEY, None)
+        if interrupted_id is None:
+            yield text_segment
+            return
+
+        raw_text_segments = self._raw_text_segments.pop(interrupted_id, None)
+        if not raw_text_segments:
+            return
+
+        raw_block = raw_text_segments[0].parent_stack[0]
+        if not self._is_inline_math(raw_block):
+            return
+
+        for raw_text_segment in raw_text_segments:
+            raw_text_segment.block_parent.attrib.pop(_ID_KEY, None)
+            yield raw_text_segment
+
+    def _has_no_math_texts(self, element: Element):
+        if element.tag == _MATH_TAG:
+            return True
+        if element.text and normalize_whitespace(element.text).strip():
+            return False
+        for child_element in element:
+            if not self._has_no_math_texts(child_element):
+                return False
+            if child_element.tail and normalize_whitespace(child_element.tail).strip():
+                return False
+        return True
+
+    def _is_inline_math(self, element: Element) -> bool:
+        if element.tag != _MATH_TAG:
+            return False
+        return element.get("display", "").lower() != "block"

epub_translator/xml_translator/__init__.py

@@ -1,3 +1,3 @@
-from .group import XMLGroupContext
-from .submitter import submit_text_segments
-from .translator import XMLTranslator
+from .callbacks import FillFailedEvent
+from .submitter import SubmitKind
+from .translator import TranslationTask, XMLTranslator

epub_translator/xml_translator/callbacks.py

@@ -0,0 +1,34 @@
+from collections.abc import Callable, Iterable
+from dataclasses import dataclass
+from xml.etree.ElementTree import Element
+
+from ..segment import TextSegment
+
+
+@dataclass
+class FillFailedEvent:
+    error_message: str
+    retried_count: int
+    over_maximum_retries: bool
+
+
+@dataclass
+class Callbacks:
+    interrupt_source_text_segments: Callable[[Iterable[TextSegment]], Iterable[TextSegment]]
+    interrupt_translated_text_segments: Callable[[Iterable[TextSegment]], Iterable[TextSegment]]
+    interrupt_block_element: Callable[[Element], Element]
+    on_fill_failed: Callable[[FillFailedEvent], None]
+
+
+def warp_callbacks(
+    interrupt_source_text_segments: Callable[[Iterable[TextSegment]], Iterable[TextSegment]] | None,
+    interrupt_translated_text_segments: Callable[[Iterable[TextSegment]], Iterable[TextSegment]] | None,
+    interrupt_block_element: Callable[[Element], Element] | None,
+    on_fill_failed: Callable[[FillFailedEvent], None] | None,
+) -> Callbacks:
+    return Callbacks(
+        interrupt_source_text_segments=interrupt_source_text_segments or (lambda x: x),
+        interrupt_translated_text_segments=interrupt_translated_text_segments or (lambda x: x),
+        interrupt_block_element=interrupt_block_element or (lambda x: x),
+        on_fill_failed=on_fill_failed or (lambda event: None),
+    )

epub_translator/xml_translator/{const.py → common.py}

@@ -1,2 +1 @@
-ID_KEY: str = "id"
 DATA_ORIGIN_LEN_KEY = "data-orig-len"

epub_translator/xml_translator/hill_climbing.py

@@ -0,0 +1,104 @@
+from collections.abc import Generator
+from dataclasses import dataclass
+from xml.etree.ElementTree import Element
+
+from tiktoken import Encoding
+
+from ..segment import BlockSegment, BlockSubmitter, TextSegment, search_text_segments
+from ..xml import plain_text
+from .common import DATA_ORIGIN_LEN_KEY
+from .stream_mapper import InlineSegmentMapping
+from .validation import LEVEL_DEPTH, generate_error_message, nest_as_errors_group, truncate_errors_group
+
+
+@dataclass
+class _BlockStatus:
+    weight: int
+    submitter: BlockSubmitter
+
+
+# 以爬山算法，将 LLM 中提交的内容中挑选出完成度更高的部分。
+# 它通过拒绝每个子部分的相对低完成度提交，锁定每个子部分只能往更高完成度的方向移动
+class HillClimbing:
+    def __init__(
+        self,
+        encoding: Encoding,
+        max_fill_displaying_errors: int,
+        block_segment: BlockSegment,
+    ) -> None:
+        self._encoding: Encoding = encoding
+        self._max_fill_displaying_errors: int = max_fill_displaying_errors
+        self._block_statuses: dict[int, _BlockStatus] = {}
+        self._block_segment: BlockSegment = block_segment
+
+    def request_element(self) -> Element:
+        element = self._block_segment.create_element()
+        for child_element in element:
+            text = plain_text(child_element)
+            tokens = self._encoding.encode(text)
+            child_element.set(DATA_ORIGIN_LEN_KEY, str(len(tokens)))
+        return element
+
+    def gen_mappings(self) -> Generator[InlineSegmentMapping | None, None, None]:
+        for inline_segment in self._block_segment:
+            id = inline_segment.id
+            assert id is not None
+            status = self._block_statuses.get(id, None)
+            text_segments: list[TextSegment] | None = None
+            if status is None:
+                yield None
+            else:
+                submitted_element = status.submitter.submitted_element
+                text_segments = list(search_text_segments(submitted_element))
+                yield inline_segment.parent, text_segments
+
+    def submit(self, element: Element) -> str | None:
+        error_message, block_weights = self._validate_block_weights_and_error_message(element)
+
+        for submitter in self._block_segment.submit(element):
+            weight: int = 0  # 未出现在 block_weights 说明没有错误，已完成
+            if block_weights:
+                weight = block_weights.get(submitter.id, 0)
+            status = self._block_statuses.get(submitter.id, None)
+            if status is None:
+                self._block_statuses[submitter.id] = _BlockStatus(
+                    weight=weight,
+                    submitter=submitter,
+                )
+            elif weight < status.weight:
+                status.weight = weight
+                status.submitter = submitter
+
+        return error_message
+
+    def _validate_block_weights_and_error_message(self, element: Element) -> tuple[str | None, dict[int, int] | None]:
+        errors_group = nest_as_errors_group(
+            errors=self._block_segment.validate(element),
+        )
+        if errors_group is None:
+            return None, None
+
+        block_weights: dict[int, int] = {}
+        for block_group in errors_group.block_groups:
+            block_id = block_group.block_id
+            status = self._block_statuses.get(block_id, None)
+            block_weights[block_id] = block_group.weight
+            if status is not None and status.weight > block_group.weight:
+                # 本轮完成度得到改善（weight 下降）应该排后，让出注意力给完成度尚未改善的部分
+                for child_error in block_group.errors:
+                    child_error.level -= LEVEL_DEPTH
+
+        origin_errors_count = errors_group.errors_count
+        errors_group = truncate_errors_group(
+            errors_group=errors_group,
+            max_errors=self._max_fill_displaying_errors,
+        )
+        if errors_group is None:
+            return None, block_weights
+
+        message = generate_error_message(
+            encoding=self._encoding,
+            errors_group=errors_group,
+            omitted_count=origin_errors_count - errors_group.errors_count,
+        )
+        return message, block_weights