epub-translator 0.0.6__py3-none-any.whl → 0.1.0__py3-none-any.whl

epub_translator/xml/firendly/decoder.py

@@ -0,0 +1,75 @@
+from collections.abc import Generator, Iterable
+from xml.etree.ElementTree import Element
+
+from ..xml import clone_element
+from .parser import parse_tags
+from .tag import Tag, TagKind
+from .transform import tag_to_element
+
+
+# why implement XML decoding?
+# https://github.com/oomol-lab/pdf-craft/issues/149
+def decode_friendly(chars: Iterable[str], tags: Iterable[str] | str = ()) -> Generator[Element, None, None]:
+    if isinstance(tags, str):
+        tags = set((tags,))
+    else:
+        tags = set(tags)
+
+    for element in _collect_elements(chars):
+        if element.tag in tags or len(tags) == 0:
+            yield clone_element(element)
+
+
+def _collect_elements(chars: Iterable[str]) -> Generator[Element, None, None]:
+    opening_stack: list[Element] = []
+    last_closed_element: Element | None = None
+
+    for cell in parse_tags(chars):
+        if isinstance(cell, Tag):
+            tag: Tag = cell
+            element = tag_to_element(tag)
+            if tag.kind == TagKind.CLOSING:
+                popped = _pop_element(tag.name, opening_stack)
+                if popped is not None:
+                    yield popped
+                    last_closed_element = popped
+                elif last_closed_element is not None:
+                    _append_to_tail(last_closed_element, tag.proto)
+            else:
+                if opening_stack:
+                    opening_stack[-1].append(element)
+                if tag.kind == TagKind.SELF_CLOSING:
+                    yield element
+                    last_closed_element = element
+                elif tag.kind == TagKind.OPENING:
+                    opening_stack.append(element)
+                    last_closed_element = None
+
+        elif last_closed_element is not None:
+            _append_to_tail(last_closed_element, cell)
+
+        elif opening_stack:
+            opening_stack[-1].text = cell
+
+
+def _append_to_tail(element: Element, text: str) -> None:
+    if element.tail:
+        element.tail += text
+    else:
+        element.tail = text
+
+
+def _pop_element(tag_name: str, opening_stack: list[Element]) -> Element | None:
+    index = -1
+    for i in range(len(opening_stack) - 1, -1, -1):
+        opening_element = opening_stack[i]
+        if tag_name == opening_element.tag:
+            index = i
+            break
+    if index == -1:
+        return None
+
+    popped: Element | None = None
+    for _ in range(len(opening_stack) - index):
+        popped = opening_stack.pop()
+    return popped

epub_translator/xml/firendly/encoder.py

@@ -0,0 +1,84 @@
+from collections.abc import Callable
+from html import escape as escape_html
+from io import StringIO
+from xml.etree.ElementTree import Element
+
+from .parser import parse_tags
+from .tag import Tag, TagKind
+from .transform import element_to_tag
+
+
+# why implement XML encoding?
+# https://github.com/oomol-lab/pdf-craft/issues/149
+def encode_friendly(element: Element, indent: int = 2) -> str:
+    buffer = StringIO()
+    _encode_element(
+        buffer=buffer,
+        element=element,
+        indent=indent,
+        depth=0,
+        escape=_escape_text,
+    )
+    return buffer.getvalue()
+
+
+def _escape_text(text: str) -> str:
+    buffer = StringIO()
+    for cell in parse_tags(text):
+        if isinstance(cell, Tag):
+            cell = escape_html(str(cell))
+        buffer.write(cell)
+    return buffer.getvalue()
+
+
+_TINY_TEXT_LEN = 35
+
+
+def _encode_element(
+    buffer: StringIO,
+    element: Element,
+    indent: int,
+    depth: int,
+    escape: Callable[[str], str],
+) -> None:
+    _write_indent(buffer, indent, depth)
+    if len(element) == 0 and not element.text:
+        tag = element_to_tag(element, TagKind.SELF_CLOSING)
+        buffer.write(str(tag))
+    else:
+        text = (element.text or "").strip()
+        opening_tag = element_to_tag(element, TagKind.OPENING)
+        closing_tag = element_to_tag(element, TagKind.CLOSING)
+        buffer.write(str(opening_tag))
+        is_one_line = len(text) <= _TINY_TEXT_LEN and len(element) == 0 and "\n" not in text
+        if text:
+            if not is_one_line:
+                buffer.write("\n")
+                _write_indent(buffer, indent, depth + 1)
+            buffer.write(escape(text))
+
+        for child in element:
+            buffer.write("\n")
+            _encode_element(
+                buffer=buffer,
+                element=child,
+                indent=indent,
+                depth=depth + 1,
+                escape=escape,
+            )
+            child_tail = (child.tail or "").strip()
+            if child_tail:
+                buffer.write("\n")
+                _write_indent(buffer, indent, depth + 1)
+                buffer.write(escape(child_tail))
+
+        if not is_one_line:
+            buffer.write("\n")
+            _write_indent(buffer, indent, depth)
+
+        buffer.write(str(closing_tag))
+
+
+def _write_indent(buffer: StringIO, indent: int, depth: int) -> None:
+    for _ in range(indent * depth):
+        buffer.write(" ")

epub_translator/xml/firendly/parser.py

@@ -0,0 +1,177 @@
+from collections.abc import Generator, Iterable
+from enum import Enum, auto
+from io import StringIO
+
+from .tag import Tag, TagKind, is_valid_name_char, is_valid_value_char
+
+_SPACES = (" ", "\n")
+
+
+class _Phase(Enum):
+    OUTSIDE = auto()
+    LEFT_BRACKET = auto()
+    LEFT_SLASH = auto()
+    TAG_NAME = auto()
+    TAG_GAP = auto()
+    ATTRIBUTE_NAME = auto()
+    ATTRIBUTE_NAME_EQUAL = auto()
+    ATTRIBUTE_VALUE = auto()
+    MUST_CLOSING_SIGN = auto()
+
+
+class _ParsedResult(Enum):
+    Continue = auto()
+    Success = auto()
+    Failed = auto()
+
+
+def parse_tags(chars: Iterable[str]) -> Generator[str | Tag, None, None]:
+    yield from _XMLTagsParser().do(chars)
+
+
+class _XMLTagsParser:
+    def __init__(self):
+        self._outside_buffer: StringIO = StringIO()
+        self._tag_buffer: StringIO = StringIO()
+        self._tag: Tag | None = None
+        self._phase: _Phase = _Phase.OUTSIDE
+
+    def do(self, chars: Iterable[str]) -> Generator[str | Tag, None, None]:
+        for char in chars:
+            parsed_result = self._parse_char(char)
+            yield from self._generate_by_result(parsed_result)
+
+        self._outside_buffer.write(self._tag_buffer.getvalue())
+        outside_text = self._outside_buffer.getvalue()
+        if outside_text != "":
+            yield outside_text
+
+    def _parse_char(self, char: str) -> _ParsedResult:
+        parsed_result: _ParsedResult = _ParsedResult.Continue
+
+        if self._phase == _Phase.OUTSIDE:
+            if char != "<":
+                self._outside_buffer.write(char)
+            else:
+                self._phase = _Phase.LEFT_BRACKET
+                self._tag_buffer.write(char)
+                self._tag = Tag(
+                    kind=TagKind.OPENING,
+                    name="",
+                    proto="",
+                    attributes=[],
+                )
+        else:
+            assert self._tag is not None
+            self._tag_buffer.write(char)
+
+            if self._phase == _Phase.LEFT_BRACKET:
+                if char == "/":
+                    self._tag.kind = TagKind.CLOSING
+                    self._phase = _Phase.LEFT_SLASH
+                elif is_valid_name_char(char):
+                    self._tag.name += char
+                    self._phase = _Phase.TAG_NAME
+                else:
+                    parsed_result = _ParsedResult.Failed
+
+            elif self._phase == _Phase.LEFT_SLASH:
+                if is_valid_name_char(char):
+                    self._tag.name += char
+                    self._phase = _Phase.TAG_NAME
+                else:
+                    parsed_result = _ParsedResult.Failed
+
+            elif self._phase == _Phase.TAG_NAME:
+                if char in _SPACES:
+                    self._phase = _Phase.TAG_GAP
+                elif is_valid_name_char(char):
+                    self._tag.name += char
+                elif char == ">":
+                    parsed_result = _ParsedResult.Success
+                elif char == "/" and self._tag.kind == TagKind.OPENING:
+                    self._tag.kind = TagKind.SELF_CLOSING
+                    self._phase = _Phase.MUST_CLOSING_SIGN
+                else:
+                    parsed_result = _ParsedResult.Failed
+
+            elif self._phase == _Phase.TAG_GAP:
+                if char in _SPACES:
+                    pass
+                elif is_valid_name_char(char):
+                    self._tag.attributes.append((char, ""))
+                    self._phase = _Phase.ATTRIBUTE_NAME
+                elif char == ">":
+                    parsed_result = _ParsedResult.Success
+                elif char == "/" and self._tag.kind == TagKind.OPENING:
+                    self._tag.kind = TagKind.SELF_CLOSING
+                    self._phase = _Phase.MUST_CLOSING_SIGN
+                else:
+                    parsed_result = _ParsedResult.Failed
+
+            elif self._phase == _Phase.ATTRIBUTE_NAME:
+                if is_valid_name_char(char):
+                    attr_name, attr_value = self._tag.attributes[-1]
+                    attr_name = attr_name + char
+                    self._tag.attributes[-1] = (attr_name, attr_value)
+                elif char == "=":
+                    self._phase = _Phase.ATTRIBUTE_NAME_EQUAL
+                else:
+                    parsed_result = _ParsedResult.Failed
+
+            elif self._phase == _Phase.ATTRIBUTE_NAME_EQUAL:
+                if char == '"':
+                    self._phase = _Phase.ATTRIBUTE_VALUE
+                else:
+                    parsed_result = _ParsedResult.Failed
+
+            elif self._phase == _Phase.ATTRIBUTE_VALUE:
+                if is_valid_value_char(char):
+                    attr_name, attr_value = self._tag.attributes[-1]
+                    attr_value = attr_value + char
+                    self._tag.attributes[-1] = (attr_name, attr_value)
+                elif char == '"':
+                    self._phase = _Phase.TAG_GAP
+                else:
+                    parsed_result = _ParsedResult.Failed
+
+            elif self._phase == _Phase.MUST_CLOSING_SIGN:
+                if char == ">":
+                    parsed_result = _ParsedResult.Success
+                else:
+                    parsed_result = _ParsedResult.Failed
+
+        return parsed_result
+
+    def _generate_by_result(self, parsed_result: _ParsedResult) -> Generator[str | Tag, None, None]:
+        if parsed_result == _ParsedResult.Success:
+            assert self._tag is not None
+            if self._is_tag_valid(self._tag):
+                outside_text = self._outside_buffer.getvalue()
+                self._clear_buffer(self._outside_buffer)
+                self._clear_buffer(self._tag_buffer)
+                if outside_text != "":
+                    yield outside_text
+                yield self._tag
+            else:
+                self._tag.proto = self._tag_buffer.getvalue()
+                self._outside_buffer.write(self._tag.proto)
+                self._clear_buffer(self._tag_buffer)
+            self._tag = None
+            self._phase = _Phase.OUTSIDE
+
+        elif parsed_result == _ParsedResult.Failed:
+            self._outside_buffer.write(self._tag_buffer.getvalue())
+            self._clear_buffer(self._tag_buffer)
+            self._phase = _Phase.OUTSIDE
+
+    def _is_tag_valid(self, tag: Tag) -> bool:
+        if tag.kind == TagKind.CLOSING and len(tag.attributes) > 0:
+            return False
+        if tag.find_invalid_name() is not None:
+            return False
+        return True
+
+    def _clear_buffer(self, buffer: StringIO):
+        buffer.truncate(0)
+        buffer.seek(0)

epub_translator/xml/firendly/tag.py

@@ -0,0 +1,118 @@
+from collections.abc import Generator
+from dataclasses import dataclass
+from enum import Enum, auto
+from io import StringIO
+
+
+class TagKind(Enum):
+    OPENING = auto()
+    CLOSING = auto()
+    SELF_CLOSING = auto()
+
+
+@dataclass
+class Tag:
+    kind: TagKind
+    name: str
+    proto: str
+    attributes: list[tuple[str, str]]
+
+    def __str__(self):
+        buffer = StringIO()
+        buffer.write("<")
+        if self.kind == TagKind.CLOSING:
+            buffer.write("/")
+        buffer.write(self.name)
+        if len(self.attributes) > 0:
+            buffer.write(" ")
+            for i, (attr_name, attr_value) in enumerate(self.attributes):
+                buffer.write(attr_name)
+                buffer.write("=")
+                buffer.write('"')
+                buffer.write(attr_value)
+                buffer.write('"')
+                if i < len(self.attributes) - 1:
+                    buffer.write(" ")
+        if self.kind == TagKind.SELF_CLOSING:
+            buffer.write("/>")
+        else:
+            buffer.write(">")
+        return buffer.getvalue()
+
+    def find_invalid_name(self) -> str | None:
+        for name in self._iter_tag_names():
+            if not all(is_valid_value_char(c) for c in name):
+                return name
+            # https://www.w3schools.com/xml/xml_elements.asp
+            # The following logic enforces a subset of XML naming rules:
+            # - Names must not be empty.
+            # - Names must start with a letter (a-z, A-Z) or an underscore (_).
+            if name == "":
+                return name
+            char = name[0]
+            if char == "_":
+                continue
+            if "a" <= char <= "z" or "A" <= char <= "Z":
+                continue
+            return name
+
+        return None
+
+    def find_invalid_attr_value(self) -> tuple[str, str] | None:
+        for attr_name, attr_value in self.attributes:
+            if not all(is_valid_value_char(c) for c in attr_value):
+                return attr_name, attr_value
+        return None
+
+    def _iter_tag_names(self) -> Generator[str, None, None]:
+        yield self.name
+        for attr_name, _ in self.attributes:
+            yield attr_name
+
+
+# XML Attribute Values: https://www.w3.org/TR/xml/#NT-AttValue
+# URI Syntax: https://www.rfc-editor.org/rfc/rfc3986
+# HTML Attributes: https://html.spec.whatwg.org/multipage/syntax.html#attributes-2
+_VALID_VALUE_CHARS = frozenset(
+    (
+        ",",
+        ".",
+        "/",
+        "#",
+        "?",
+        "&",
+        "=",
+        ":",
+        "%",
+        ";",
+        " ",
+    )
+)
+
+
+# XML Names: https://www.w3.org/TR/xml/#NT-Name
+# XML Namespaces: https://www.w3.org/TR/xml-names/#ns-qualnames
+# HTML Custom Data Attributes: https://html.spec.whatwg.org/multipage/dom.html#custom-data-attribute
+_VALID_NAME_CHARS = frozenset(("-", "_", ":", "."))
+
+
+def is_valid_value_char(char: str) -> bool:
+    if is_valid_name_char(char):
+        return True
+    if char in _VALID_VALUE_CHARS:
+        return True
+    return False
+
+
+def is_valid_name_char(char: str) -> bool:
+    if char in _VALID_NAME_CHARS:
+        return True
+
+    # https://www.w3.org/TR/xml/#NT-Name
+    if "a" <= char <= "z":
+        return True
+    if "A" <= char <= "Z":
+        return True
+    if "0" <= char <= "9":
+        return True
+    return False

epub_translator/xml/firendly/transform.py

@@ -0,0 +1,36 @@
+from xml.etree.ElementTree import Element
+
+from .tag import Tag, TagKind
+
+
+def tag_to_element(tag: Tag) -> Element:
+    element = Element(tag.name)
+    for attr_name, attr_value in tag.attributes:
+        element.set(attr_name, attr_value)
+    return element
+
+
+def element_to_tag(element: Element, kind: TagKind, proto: str = "") -> Tag:
+    tag = Tag(
+        kind=kind,
+        name=element.tag,
+        proto=proto,
+        attributes=[],
+    )
+    if kind != TagKind.CLOSING:
+        for attr_name in sorted(list(element.keys())):
+            attr_value = element.get(attr_name, "")
+            tag.attributes.append((attr_name, attr_value))
+
+    # To make LLM easier to understand, the naming here is restricted in a more strict way.
+    # https://github.com/oomol-lab/pdf-craft/issues/149
+    invalid_name = tag.find_invalid_name()
+    if invalid_name is not None:
+        raise ValueError(f"find invalid tag name or attribute name: {invalid_name}")
+
+    invalid_attr_pair = tag.find_invalid_attr_value()
+    if invalid_attr_pair is not None:
+        attr_name, attr_value = invalid_attr_pair
+        raise ValueError(f'find invalid attribute value: {attr_name}="{attr_value}"')
+
+    return tag

epub_translator/xml/xml.py

@@ -0,0 +1,52 @@
+from collections.abc import Generator
+from xml.etree.ElementTree import Element
+
+
+def find_first(element: Element, tag: str) -> Element | None:
+    if element.tag == tag:
+        return element
+    for child in element:
+        result = find_first(child, tag)
+        if result is not None:
+            return result
+    return None
+
+
+def iter_with_stack(element: Element) -> Generator[tuple[list[Element], Element], None, None]:
+    """先序遍历：yield parent_path, element"""
+    stack: list[list[Element]] = [[element]]
+    while stack:
+        current_path = stack.pop()
+        current = current_path[-1]
+        yield current_path[:-1], current
+
+        if len(current) == 0:
+            continue
+
+        for child in reversed(list(current)):
+            child_path = list(current_path)
+            child_path.append(child)
+            stack.append(child_path)
+
+
+def clone_element(element: Element) -> Element:
+    new_element = Element(element.tag, element.attrib)
+    new_element.text = element.text
+    for child in element:
+        new_child = clone_element(child)
+        new_child.tail = child.tail
+        new_element.append(new_child)
+    return new_element
+
+
+def plain_text(element: Element) -> str:
+    return "".join(_iter_text_in(element))
+
+
+def _iter_text_in(element: Element) -> Generator[str, None, None]:
+    if element.text:
+        yield element.text
+    for child in element:
+        yield from _iter_text_in(child)
+        if child.tail:
+            yield child.tail