PyPI - epub-translator - Versions diffs - 0.1.1__py3-none-any.whl → 0.1.3__py3-none-any.whl - Mend

epub-translator 0.1.1py3-none-any.whl → 0.1.3py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Files changed (50) hide show

epub_translator/__init__.py +2 -2
epub_translator/data/fill.jinja +143 -38
epub_translator/epub/__init__.py +1 -1
epub_translator/epub/metadata.py +122 -0
epub_translator/epub/spines.py +3 -2
epub_translator/epub/zip.py +11 -9
epub_translator/epub_transcode.py +108 -0
epub_translator/llm/__init__.py +1 -0
epub_translator/llm/context.py +109 -0
epub_translator/llm/core.py +32 -113
epub_translator/llm/executor.py +25 -31
epub_translator/llm/increasable.py +1 -1
epub_translator/llm/types.py +0 -3
epub_translator/segment/__init__.py +26 -0
epub_translator/segment/block_segment.py +124 -0
epub_translator/segment/common.py +29 -0
epub_translator/segment/inline_segment.py +356 -0
epub_translator/{xml_translator → segment}/text_segment.py +8 -8
epub_translator/segment/utils.py +43 -0
epub_translator/translator.py +147 -183
epub_translator/utils.py +33 -0
epub_translator/xml/__init__.py +2 -0
epub_translator/xml/const.py +1 -0
epub_translator/xml/deduplication.py +3 -3
epub_translator/xml/self_closing.py +182 -0
epub_translator/xml/utils.py +42 -0
epub_translator/xml/xml.py +7 -0
epub_translator/xml/xml_like.py +8 -33
epub_translator/xml_interrupter.py +165 -0
epub_translator/xml_translator/__init__.py +1 -2
epub_translator/xml_translator/callbacks.py +34 -0
epub_translator/xml_translator/{const.py → common.py} +0 -1
epub_translator/xml_translator/hill_climbing.py +104 -0
epub_translator/xml_translator/stream_mapper.py +253 -0
epub_translator/xml_translator/submitter.py +26 -72
epub_translator/xml_translator/translator.py +162 -113
epub_translator/xml_translator/validation.py +458 -0
{epub_translator-0.1.1.dist-info → epub_translator-0.1.3.dist-info}/METADATA +72 -9
epub_translator-0.1.3.dist-info/RECORD +66 -0
epub_translator/epub/placeholder.py +0 -53
epub_translator/iter_sync.py +0 -24
epub_translator/xml_translator/fill.py +0 -128
epub_translator/xml_translator/format.py +0 -282
epub_translator/xml_translator/fragmented.py +0 -125
epub_translator/xml_translator/group.py +0 -183
epub_translator/xml_translator/progressive_locking.py +0 -256
epub_translator/xml_translator/utils.py +0 -29
epub_translator-0.1.1.dist-info/RECORD +0 -58
{epub_translator-0.1.1.dist-info → epub_translator-0.1.3.dist-info}/LICENSE +0 -0
{epub_translator-0.1.1.dist-info → epub_translator-0.1.3.dist-info}/WHEEL +0 -0

epub_translator/xml/xml_like.py CHANGED Viewed

@@ -4,6 +4,7 @@ import warnings
 from typing import IO
 from xml.etree.ElementTree import Element, fromstring, tostring
+from .self_closing import self_close_void_elements, unclose_void_elements
 from .xml import iter_with_stack
 _XML_NAMESPACE_URI = "http://www.w3.org/XML/1998/namespace"
@@ -31,28 +32,11 @@ _ENCODING_PATTERN = re.compile(r'encoding\s*=\s*["\']([^"\']+)["\']', re.IGNOREC
 _FIRST_ELEMENT_PATTERN = re.compile(r"<(?![?!])[a-zA-Z]")
 _NAMESPACE_IN_TAG = re.compile(r"\{([^}]+)\}")
-# Some non-standard EPUB generators use HTML-style tags without self-closing syntax
-# We need to convert them to XML-compatible format before parsing
-_EMPTY_TAGS = (
-    "br",
-    "hr",
-    "input",
-    "col",
-    "base",
-    "meta",
-    "area",
-)
-# For reading: match tags like <br> or <br class="x"> (but not <br/> or <body>)
-_EMPTY_TAG_OPEN_PATTERN = re.compile(r"<(" + "|".join(_EMPTY_TAGS) + r")(\s[^/>]*)>")
-# For saving: match self-closing tags like <br />
-_EMPTY_TAG_CLOSE_PATTERN = re.compile(r"<(" + "|".join(_EMPTY_TAGS) + r")(\s[^>]*?)\s*/>")
 class XMLLikeNode:
     def __init__(self, file: IO[bytes], is_html_like: bool = False) -> None:
         raw_content = file.read()
+        self._is_html_like = is_html_like
         self._encoding: str = self._detect_encoding(raw_content)
         content = raw_content.decode(self._encoding)
         self._header, xml_content = self._extract_header(content)
@@ -60,16 +44,9 @@ class XMLLikeNode:
         self._tag_to_namespace: dict[str, str] = {}
         self._attr_to_namespace: dict[str, str] = {}
-        # For non-standard HTML files, convert <br> to <br/> before parsing
-        self._is_html_like = is_html_like
-        if is_html_like:
-            xml_content = re.sub(
-                pattern=_EMPTY_TAG_OPEN_PATTERN,
-                repl=lambda m: f"<{m.group(1)}{m.group(2)} />",
-                string=xml_content,
-            )
         try:
+            # 不必判断类型，这是一个防御性极强的函数，可做到 shit >> XML
+            xml_content = self_close_void_elements(xml_content)
             self.element = self._extract_and_clean_namespaces(
                 element=fromstring(xml_content),
             )
@@ -92,13 +69,11 @@ class XMLLikeNode:
             content = self._serialize_with_namespaces(self.element)
-            # For non-standard HTML files, convert back from <br/> to <br>
+            # For non-standard HTML files (text/html), convert back from <br/> to <br>
+            # to maintain compatibility with HTML parsers that don't support XHTML
+            # For XHTML files (application/xhtml+xml), keep self-closing format
             if self._is_html_like:
-                content = re.sub(
-                    pattern=_EMPTY_TAG_CLOSE_PATTERN,
-                    repl=lambda m: f"<{m.group(1)}{m.group(2)}>",
-                    string=content,
-                )
+                content = unclose_void_elements(content)
             writer.write(content)

epub_translator/xml_interrupter.py ADDED Viewed

@@ -0,0 +1,165 @@
+from collections.abc import Generator, Iterable
+from typing import cast
+from xml.etree.ElementTree import Element
+from .segment import TextSegment
+from .utils import ensure_list, normalize_whitespace
+_ID_KEY = "__XML_INTERRUPTER_ID"
+_MATH_TAG = "math"
+_EXPRESSION_TAG = "expression"
+class XMLInterrupter:
+    def __init__(self) -> None:
+        self._next_id: int = 1
+        self._last_interrupted_id: str | None = None
+        self._placeholder2interrupted: dict[int, Element] = {}
+        self._raw_text_segments: dict[str, list[TextSegment]] = {}
+    def interrupt_source_text_segments(
+        self, text_segments: Iterable[TextSegment]
+    ) -> Generator[TextSegment, None, None]:
+        for text_segment in text_segments:
+            yield from self._expand_source_text_segment(text_segment)
+        if self._last_interrupted_id is not None:
+            merged_text_segment = self._pop_and_merge_from_buffered(self._last_interrupted_id)
+            if merged_text_segment:
+                yield merged_text_segment
+    def interrupt_translated_text_segments(
+        self, text_segments: Iterable[TextSegment]
+    ) -> Generator[TextSegment, None, None]:
+        for text_segment in text_segments:
+            yield from self._expand_translated_text_segment(text_segment)
+    def interrupt_block_element(self, element: Element) -> Element:
+        interrupted_element = self._placeholder2interrupted.pop(id(element), None)
+        if interrupted_element is None:
+            return element
+        else:
+            return interrupted_element
+    def _expand_source_text_segment(self, text_segment: TextSegment):
+        interrupted_index = self._interrupted_index(text_segment)
+        interrupted_id: str | None = None
+        if interrupted_index is not None:
+            interrupted_element = text_segment.parent_stack[interrupted_index]
+            interrupted_id = interrupted_element.get(_ID_KEY)
+            if interrupted_id is None:
+                interrupted_id = str(self._next_id)
+                interrupted_element.set(_ID_KEY, interrupted_id)
+                self._next_id += 1
+            text_segments = ensure_list(
+                target=self._raw_text_segments,
+                key=interrupted_id,
+            )
+            text_segments.append(text_segment)
+        if self._last_interrupted_id is not None and interrupted_id != self._last_interrupted_id:
+            merged_text_segment = self._pop_and_merge_from_buffered(self._last_interrupted_id)
+            if merged_text_segment:
+                yield merged_text_segment
+        self._last_interrupted_id = interrupted_id
+        if interrupted_index is None:
+            yield text_segment
+    def _pop_and_merge_from_buffered(self, interrupted_id: str) -> TextSegment | None:
+        merged_text_segment: TextSegment | None = None
+        text_segments = self._raw_text_segments.get(interrupted_id, None)
+        if text_segments:
+            text_segment = text_segments[0]
+            interrupted_index = cast(int, self._interrupted_index(text_segment))
+            interrupted_element = text_segment.parent_stack[cast(int, interrupted_index)]
+            placeholder_element = Element(
+                _EXPRESSION_TAG,
+                {
+                    _ID_KEY: cast(str, interrupted_element.get(_ID_KEY)),
+                },
+            )
+            raw_parent_stack = text_segment.parent_stack[:interrupted_index]
+            parent_stack = raw_parent_stack + [placeholder_element]
+            merged_text_segment = TextSegment(
+                text="".join(t.text for t in text_segments),
+                parent_stack=parent_stack,
+                left_common_depth=text_segments[0].left_common_depth,
+                right_common_depth=text_segments[-1].right_common_depth,
+                block_depth=len(parent_stack),
+                position=text_segments[0].position,
+            )
+            self._placeholder2interrupted[id(placeholder_element)] = interrupted_element
+            # TODO: 比较难搞，先关了再说
+            # parent_element: Element | None = None
+            # if interrupted_index > 0:
+            #     parent_element = text_segment.parent_stack[interrupted_index - 1]
+            # if (
+            #     not self._is_inline_math(interrupted_element)
+            #     or parent_element is None
+            #     or self._has_no_math_texts(parent_element)
+            # ):
+            #     # 区块级公式不必重复出现，出现时突兀。但行内公式穿插在译文中更有利于读者阅读顺畅。
+            #     self._raw_text_segments.pop(interrupted_id, None)
+            # else:
+            #     for text_segment in text_segments:
+            #         # 原始栈退光，仅留下相对 interrupted 元素的栈，这种格式与 translated 要求一致
+            #         text_segment.left_common_depth = max(0, text_segment.left_common_depth - interrupted_index)
+            #         text_segment.right_common_depth = max(0, text_segment.right_common_depth - interrupted_index)
+            #         text_segment.block_depth = 1
+            #         text_segment.parent_stack = text_segment.parent_stack[interrupted_index:]
+            for text_segment in text_segments:
+                # 原始栈退光，仅留下相对 interrupted 元素的栈，这种格式与 translated 要求一致
+                text_segment.left_common_depth = max(0, text_segment.left_common_depth - interrupted_index)
+                text_segment.right_common_depth = max(0, text_segment.right_common_depth - interrupted_index)
+                text_segment.block_depth = 1
+                text_segment.parent_stack = text_segment.parent_stack[interrupted_index:]
+        return merged_text_segment
+    def _interrupted_index(self, text_segment: TextSegment) -> int | None:
+        interrupted_index: int | None = None
+        for i, parent_element in enumerate(text_segment.parent_stack):
+            if parent_element.tag == _MATH_TAG:
+                interrupted_index = i
+                break
+        return interrupted_index
+    def _expand_translated_text_segment(self, text_segment: TextSegment):
+        interrupted_id = text_segment.block_parent.attrib.pop(_ID_KEY, None)
+        if interrupted_id is None:
+            yield text_segment
+            return
+        raw_text_segments = self._raw_text_segments.pop(interrupted_id, None)
+        if not raw_text_segments:
+            return
+        raw_block = raw_text_segments[0].parent_stack[0]
+        if not self._is_inline_math(raw_block):
+            return
+        for raw_text_segment in raw_text_segments:
+            raw_text_segment.block_parent.attrib.pop(_ID_KEY, None)
+            yield raw_text_segment
+    def _has_no_math_texts(self, element: Element):
+        if element.tag == _MATH_TAG:
+            return True
+        if element.text and normalize_whitespace(element.text).strip():
+            return False
+        for child_element in element:
+            if not self._has_no_math_texts(child_element):
+                return False
+            if child_element.tail and normalize_whitespace(child_element.tail).strip():
+                return False
+        return True
+    def _is_inline_math(self, element: Element) -> bool:
+        if element.tag != _MATH_TAG:
+            return False
+        return element.get("display", "").lower() != "block"

epub_translator/xml_translator/__init__.py CHANGED Viewed

@@ -1,3 +1,2 @@
-from .group import XMLGroupContext
-from .submitter import submit_text_segments
+from .callbacks import FillFailedEvent
 from .translator import XMLTranslator

epub_translator/xml_translator/callbacks.py ADDED Viewed

@@ -0,0 +1,34 @@
+from collections.abc import Callable, Iterable
+from dataclasses import dataclass
+from xml.etree.ElementTree import Element
+from ..segment import TextSegment
+@dataclass
+class FillFailedEvent:
+    error_message: str
+    retried_count: int
+    over_maximum_retries: bool
+@dataclass
+class Callbacks:
+    interrupt_source_text_segments: Callable[[Iterable[TextSegment]], Iterable[TextSegment]]
+    interrupt_translated_text_segments: Callable[[Iterable[TextSegment]], Iterable[TextSegment]]
+    interrupt_block_element: Callable[[Element], Element]
+    on_fill_failed: Callable[[FillFailedEvent], None]
+def warp_callbacks(
+    interrupt_source_text_segments: Callable[[Iterable[TextSegment]], Iterable[TextSegment]] | None,
+    interrupt_translated_text_segments: Callable[[Iterable[TextSegment]], Iterable[TextSegment]] | None,
+    interrupt_block_element: Callable[[Element], Element] | None,
+    on_fill_failed: Callable[[FillFailedEvent], None] | None,
+) -> Callbacks:
+    return Callbacks(
+        interrupt_source_text_segments=interrupt_source_text_segments or (lambda x: x),
+        interrupt_translated_text_segments=interrupt_translated_text_segments or (lambda x: x),
+        interrupt_block_element=interrupt_block_element or (lambda x: x),
+        on_fill_failed=on_fill_failed or (lambda event: None),
+    )

epub_translator/xml_translator/{const.py → common.py} RENAMED Viewed

	@@ -1,2 +1 @@
1	- ID_KEY: str = "id"
2 1	DATA_ORIGIN_LEN_KEY = "data-orig-len"

epub_translator/xml_translator/hill_climbing.py ADDED Viewed

@@ -0,0 +1,104 @@
+from collections.abc import Generator
+from dataclasses import dataclass
+from xml.etree.ElementTree import Element
+from tiktoken import Encoding
+from ..segment import BlockSegment, BlockSubmitter, TextSegment, search_text_segments
+from ..xml import plain_text
+from .common import DATA_ORIGIN_LEN_KEY
+from .stream_mapper import InlineSegmentMapping
+from .validation import LEVEL_DEPTH, generate_error_message, nest_as_errors_group, truncate_errors_group
+@dataclass
+class _BlockStatus:
+    weight: int
+    submitter: BlockSubmitter
+# 以爬山算法，将 LLM 中提交的内容中挑选出完成度更高的部分。
+# 它通过拒绝每个子部分的相对低完成度提交，锁定每个子部分只能往更高完成度的方向移动
+class HillClimbing:
+    def __init__(
+        self,
+        encoding: Encoding,
+        max_fill_displaying_errors: int,
+        block_segment: BlockSegment,
+    ) -> None:
+        self._encoding: Encoding = encoding
+        self._max_fill_displaying_errors: int = max_fill_displaying_errors
+        self._block_statuses: dict[int, _BlockStatus] = {}
+        self._block_segment: BlockSegment = block_segment
+    def request_element(self) -> Element:
+        element = self._block_segment.create_element()
+        for child_element in element:
+            text = plain_text(child_element)
+            tokens = self._encoding.encode(text)
+            child_element.set(DATA_ORIGIN_LEN_KEY, str(len(tokens)))
+        return element
+    def gen_mappings(self) -> Generator[InlineSegmentMapping | None, None, None]:
+        for inline_segment in self._block_segment:
+            id = inline_segment.id
+            assert id is not None
+            status = self._block_statuses.get(id, None)
+            text_segments: list[TextSegment] | None = None
+            if status is None:
+                yield None
+            else:
+                submitted_element = status.submitter.submitted_element
+                text_segments = list(search_text_segments(submitted_element))
+                yield inline_segment.parent, text_segments
+    def submit(self, element: Element) -> str | None:
+        error_message, block_weights = self._validate_block_weights_and_error_message(element)
+        for submitter in self._block_segment.submit(element):
+            weight: int = 0  # 未出现在 block_weights 说明没有错误，已完成
+            if block_weights:
+                weight = block_weights.get(submitter.id, 0)
+            status = self._block_statuses.get(submitter.id, None)
+            if status is None:
+                self._block_statuses[submitter.id] = _BlockStatus(
+                    weight=weight,
+                    submitter=submitter,
+                )
+            elif weight < status.weight:
+                status.weight = weight
+                status.submitter = submitter
+        return error_message
+    def _validate_block_weights_and_error_message(self, element: Element) -> tuple[str | None, dict[int, int] | None]:
+        errors_group = nest_as_errors_group(
+            errors=self._block_segment.validate(element),
+        )
+        if errors_group is None:
+            return None, None
+        block_weights: dict[int, int] = {}
+        for block_group in errors_group.block_groups:
+            block_id = block_group.block_id
+            status = self._block_statuses.get(block_id, None)
+            block_weights[block_id] = block_group.weight
+            if status is not None and status.weight > block_group.weight:
+                # 本轮完成度得到改善（weight 下降）应该排后，让出注意力给完成度尚未改善的部分
+                for child_error in block_group.errors:
+                    child_error.level -= LEVEL_DEPTH
+        origin_errors_count = errors_group.errors_count
+        errors_group = truncate_errors_group(
+            errors_group=errors_group,
+            max_errors=self._max_fill_displaying_errors,
+        )
+        if errors_group is None:
+            return None, block_weights
+        message = generate_error_message(
+            encoding=self._encoding,
+            errors_group=errors_group,
+            omitted_count=origin_errors_count - errors_group.errors_count,
+        )
+        return message, block_weights

epub_translator/xml_translator/stream_mapper.py ADDED Viewed

@@ -0,0 +1,253 @@
+from collections.abc import Callable, Generator, Iterable, Iterator
+from xml.etree.ElementTree import Element
+from resource_segmentation import Group, Resource, Segment, split
+from tiktoken import Encoding
+from ..segment import InlineSegment, TextSegment, search_inline_segments, search_text_segments
+from .callbacks import Callbacks
+_PAGE_INCISION = 0
+_BLOCK_INCISION = 1
+_ELLIPSIS = "..."
+InlineSegmentMapping = tuple[Element, list[TextSegment]]
+InlineSegmentGroupMap = Callable[[list[InlineSegment]], list[InlineSegmentMapping | None]]
+class XMLStreamMapper:
+    def __init__(self, encoding: Encoding, max_group_tokens: int) -> None:
+        self._encoding: Encoding = encoding
+        self._max_group_tokens: int = max_group_tokens
+    def map_stream(
+        self,
+        elements: Iterator[Element],
+        callbacks: Callbacks,
+        map: InlineSegmentGroupMap,
+    ) -> Generator[tuple[Element, list[InlineSegmentMapping]], None, None]:
+        current_element: Element | None = None
+        mapping_buffer: list[InlineSegmentMapping] = []
+        for group in self._split_into_serial_groups(elements, callbacks):
+            head, body, tail = self._truncate_and_transform_group(group)
+            target_body = map(head + body + tail)[len(head) : len(head) + len(body)]
+            for origin, target in zip(body, target_body, strict=False):
+                origin_element = origin.head.root
+                if current_element is None:
+                    current_element = origin_element
+                if id(current_element) != id(origin_element):
+                    yield current_element, mapping_buffer
+                    current_element = origin_element
+                    mapping_buffer = []
+                if target:
+                    block_element, text_segments = target
+                    block_element = callbacks.interrupt_block_element(block_element)
+                    text_segments = list(callbacks.interrupt_translated_text_segments(text_segments))
+                    if text_segments:
+                        mapping_buffer.append((block_element, text_segments))
+        if current_element is not None:
+            yield current_element, mapping_buffer
+    def _split_into_serial_groups(self, elements: Iterable[Element], callbacks: Callbacks):
+        def generate():
+            for element in elements:
+                yield from split(
+                    max_segment_count=self._max_group_tokens,
+                    border_incision=_PAGE_INCISION,
+                    resources=self._expand_to_resources(element, callbacks),
+                )
+        generator = generate()
+        group = next(generator, None)
+        if group is None:
+            return
+        # head + body * N (without tail)
+        sum_count = group.head_remain_count + sum(x.count for x in self._expand_resource_segments(group.body))
+        while True:
+            next_group = next(generator, None)
+            if next_group is None:
+                break
+            next_sum_body_count = sum(x.count for x in self._expand_resource_segments(next_group.body))
+            next_sum_count = sum_count + next_sum_body_count
+            if next_sum_count + next_group.tail_remain_count > self._max_group_tokens:
+                yield group
+                group = next_group
+                sum_count = group.head_remain_count + next_sum_body_count
+            else:
+                group.body.extend(next_group.body)
+                group.tail = next_group.tail
+                group.tail_remain_count = next_group.tail_remain_count
+                sum_count = next_sum_count
+        yield group
+    def _truncate_and_transform_group(self, group: Group[InlineSegment]):
+        head = list(
+            self._truncate_inline_segments(
+                inline_segments=self._expand_inline_segments(group.head),
+                remain_head=False,
+                remain_count=group.head_remain_count,
+            )
+        )
+        body = list(self._expand_inline_segments(group.body))
+        tail = list(
+            self._truncate_inline_segments(
+                inline_segments=self._expand_inline_segments(group.tail),
+                remain_head=True,
+                remain_count=group.tail_remain_count,
+            )
+        )
+        return head, body, tail
+    def _expand_to_resources(self, element: Element, callbacks: Callbacks):
+        def expand(element: Element):
+            text_segments = search_text_segments(element)
+            text_segments = callbacks.interrupt_source_text_segments(text_segments)
+            yield from search_inline_segments(text_segments)
+        inline_segment_generator = expand(element)
+        start_incision = _PAGE_INCISION
+        inline_segment = next(inline_segment_generator, None)
+        if inline_segment is None:
+            return
+        while True:
+            next_inline_segment = next(inline_segment_generator, None)
+            if next_inline_segment is None:
+                break
+            if next_inline_segment.head.root is inline_segment.tail.root:
+                end_incision = _BLOCK_INCISION
+            else:
+                end_incision = _PAGE_INCISION
+            yield Resource(
+                count=sum(len(self._encoding.encode(t.xml_text)) for t in inline_segment),
+                start_incision=start_incision,
+                end_incision=end_incision,
+                payload=inline_segment,
+            )
+            inline_segment = next_inline_segment
+            start_incision = end_incision
+        yield Resource(
+            count=sum(len(self._encoding.encode(t.xml_text)) for t in inline_segment),
+            start_incision=start_incision,
+            end_incision=_PAGE_INCISION,
+            payload=inline_segment,
+        )
+    def _truncate_inline_segments(self, inline_segments: Iterable[InlineSegment], remain_head: bool, remain_count: int):
+        def clone_and_expand(segments: Iterable[InlineSegment]):
+            for segment in segments:
+                for child_segment in segment:
+                    yield child_segment.clone()  # 切割对应的 head 和 tail 会与其他 group 重叠，复制避免互相影响
+        truncated_text_segments = self._truncate_text_segments(
+            text_segments=clone_and_expand(inline_segments),
+            remain_head=remain_head,
+            remain_count=remain_count,
+        )
+        yield from search_inline_segments(truncated_text_segments)
+    def _expand_inline_segments(self, items: list[Resource[InlineSegment] | Segment[InlineSegment]]):
+        for resource in self._expand_resource_segments(items):
+            yield resource.payload
+    def _expand_resource_segments(self, items: list[Resource[InlineSegment] | Segment[InlineSegment]]):
+        for item in items:
+            if isinstance(item, Resource):
+                yield item
+            elif isinstance(item, Segment):
+                yield from item.resources
+    def _truncate_text_segments(self, text_segments: Iterable[TextSegment], remain_head: bool, remain_count: int):
+        if remain_head:
+            yield from self._filter_and_remain_segments(
+                segments=text_segments,
+                remain_head=remain_head,
+                remain_count=remain_count,
+            )
+        else:
+            yield from reversed(
+                list(
+                    self._filter_and_remain_segments(
+                        segments=reversed(list(text_segments)),
+                        remain_head=remain_head,
+                        remain_count=remain_count,
+                    )
+                )
+            )
+    def _filter_and_remain_segments(self, segments: Iterable[TextSegment], remain_head: bool, remain_count: int):
+        for segment in segments:
+            if remain_count <= 0:
+                break
+            raw_xml_text = segment.xml_text
+            tokens = self._encoding.encode(raw_xml_text)
+            tokens_count = len(tokens)
+            if tokens_count > remain_count:
+                truncated_segment = self._truncate_text_segment(
+                    segment=segment,
+                    tokens=tokens,
+                    raw_xml_text=raw_xml_text,
+                    remain_head=remain_head,
+                    remain_count=remain_count,
+                )
+                if truncated_segment is not None:
+                    yield truncated_segment
+                break
+            yield segment
+            remain_count -= tokens_count
+    def _truncate_text_segment(
+        self,
+        segment: TextSegment,
+        tokens: list[int],
+        raw_xml_text: str,
+        remain_head: bool,
+        remain_count: int,
+    ) -> TextSegment | None:
+        # 典型的 xml_text: <tag id="99" data-origin-len="999">Some text</tag>
+        # 如果切割点在前缀 XML 区，则整体舍弃
+        # 如果切割点在后缀 XML 区，则整体保留
+        # 只有刚好切割在正文区，才执行文本截断操作
+        remain_text: str
+        xml_text_head_length = raw_xml_text.find(segment.text)
+        if remain_head:
+            remain_xml_text = self._encoding.decode(tokens[:remain_count])  # remain_count cannot be 0 here
+            if len(remain_xml_text) <= xml_text_head_length:
+                return None
+            if len(remain_xml_text) >= xml_text_head_length + len(segment.text):
+                return segment
+            remain_text = remain_xml_text[xml_text_head_length:]
+        else:
+            xml_text_tail_length = len(raw_xml_text) - (xml_text_head_length + len(segment.text))
+            remain_xml_text = self._encoding.decode(tokens[-remain_count:])
+            if len(remain_xml_text) <= xml_text_tail_length:
+                return None
+            if len(remain_xml_text) >= xml_text_tail_length + len(segment.text):
+                return segment
+            remain_text = remain_xml_text[: len(remain_xml_text) - xml_text_tail_length]
+        if not remain_text.strip():
+            return None
+        if remain_head:
+            segment.text = f"{remain_text} {_ELLIPSIS}"
+        else:
+            segment.text = f"{_ELLIPSIS} {remain_text}"
+        return segment

epub-translator 0.1.1__py3-none-any.whl → 0.1.3__py3-none-any.whl

epub-translator 0.1.1py3-none-any.whl → 0.1.3py3-none-any.whl