epub-translator 0.0.7__py3-none-any.whl → 0.1.1__py3-none-any.whl

epub_translator/xml/firendly/parser.py

@@ -0,0 +1,177 @@
+from collections.abc import Generator, Iterable
+from enum import Enum, auto
+from io import StringIO
+
+from .tag import Tag, TagKind, is_valid_name_char, is_valid_value_char
+
+_SPACES = (" ", "\n")
+
+
+class _Phase(Enum):
+    OUTSIDE = auto()
+    LEFT_BRACKET = auto()
+    LEFT_SLASH = auto()
+    TAG_NAME = auto()
+    TAG_GAP = auto()
+    ATTRIBUTE_NAME = auto()
+    ATTRIBUTE_NAME_EQUAL = auto()
+    ATTRIBUTE_VALUE = auto()
+    MUST_CLOSING_SIGN = auto()
+
+
+class _ParsedResult(Enum):
+    Continue = auto()
+    Success = auto()
+    Failed = auto()
+
+
+def parse_tags(chars: Iterable[str]) -> Generator[str | Tag, None, None]:
+    yield from _XMLTagsParser().do(chars)
+
+
+class _XMLTagsParser:
+    def __init__(self):
+        self._outside_buffer: StringIO = StringIO()
+        self._tag_buffer: StringIO = StringIO()
+        self._tag: Tag | None = None
+        self._phase: _Phase = _Phase.OUTSIDE
+
+    def do(self, chars: Iterable[str]) -> Generator[str | Tag, None, None]:
+        for char in chars:
+            parsed_result = self._parse_char(char)
+            yield from self._generate_by_result(parsed_result)
+
+        self._outside_buffer.write(self._tag_buffer.getvalue())
+        outside_text = self._outside_buffer.getvalue()
+        if outside_text != "":
+            yield outside_text
+
+    def _parse_char(self, char: str) -> _ParsedResult:
+        parsed_result: _ParsedResult = _ParsedResult.Continue
+
+        if self._phase == _Phase.OUTSIDE:
+            if char != "<":
+                self._outside_buffer.write(char)
+            else:
+                self._phase = _Phase.LEFT_BRACKET
+                self._tag_buffer.write(char)
+                self._tag = Tag(
+                    kind=TagKind.OPENING,
+                    name="",
+                    proto="",
+                    attributes=[],
+                )
+        else:
+            assert self._tag is not None
+            self._tag_buffer.write(char)
+
+            if self._phase == _Phase.LEFT_BRACKET:
+                if char == "/":
+                    self._tag.kind = TagKind.CLOSING
+                    self._phase = _Phase.LEFT_SLASH
+                elif is_valid_name_char(char):
+                    self._tag.name += char
+                    self._phase = _Phase.TAG_NAME
+                else:
+                    parsed_result = _ParsedResult.Failed
+
+            elif self._phase == _Phase.LEFT_SLASH:
+                if is_valid_name_char(char):
+                    self._tag.name += char
+                    self._phase = _Phase.TAG_NAME
+                else:
+                    parsed_result = _ParsedResult.Failed
+
+            elif self._phase == _Phase.TAG_NAME:
+                if char in _SPACES:
+                    self._phase = _Phase.TAG_GAP
+                elif is_valid_name_char(char):
+                    self._tag.name += char
+                elif char == ">":
+                    parsed_result = _ParsedResult.Success
+                elif char == "/" and self._tag.kind == TagKind.OPENING:
+                    self._tag.kind = TagKind.SELF_CLOSING
+                    self._phase = _Phase.MUST_CLOSING_SIGN
+                else:
+                    parsed_result = _ParsedResult.Failed
+
+            elif self._phase == _Phase.TAG_GAP:
+                if char in _SPACES:
+                    pass
+                elif is_valid_name_char(char):
+                    self._tag.attributes.append((char, ""))
+                    self._phase = _Phase.ATTRIBUTE_NAME
+                elif char == ">":
+                    parsed_result = _ParsedResult.Success
+                elif char == "/" and self._tag.kind == TagKind.OPENING:
+                    self._tag.kind = TagKind.SELF_CLOSING
+                    self._phase = _Phase.MUST_CLOSING_SIGN
+                else:
+                    parsed_result = _ParsedResult.Failed
+
+            elif self._phase == _Phase.ATTRIBUTE_NAME:
+                if is_valid_name_char(char):
+                    attr_name, attr_value = self._tag.attributes[-1]
+                    attr_name = attr_name + char
+                    self._tag.attributes[-1] = (attr_name, attr_value)
+                elif char == "=":
+                    self._phase = _Phase.ATTRIBUTE_NAME_EQUAL
+                else:
+                    parsed_result = _ParsedResult.Failed
+
+            elif self._phase == _Phase.ATTRIBUTE_NAME_EQUAL:
+                if char == '"':
+                    self._phase = _Phase.ATTRIBUTE_VALUE
+                else:
+                    parsed_result = _ParsedResult.Failed
+
+            elif self._phase == _Phase.ATTRIBUTE_VALUE:
+                if is_valid_value_char(char):
+                    attr_name, attr_value = self._tag.attributes[-1]
+                    attr_value = attr_value + char
+                    self._tag.attributes[-1] = (attr_name, attr_value)
+                elif char == '"':
+                    self._phase = _Phase.TAG_GAP
+                else:
+                    parsed_result = _ParsedResult.Failed
+
+            elif self._phase == _Phase.MUST_CLOSING_SIGN:
+                if char == ">":
+                    parsed_result = _ParsedResult.Success
+                else:
+                    parsed_result = _ParsedResult.Failed
+
+        return parsed_result
+
+    def _generate_by_result(self, parsed_result: _ParsedResult) -> Generator[str | Tag, None, None]:
+        if parsed_result == _ParsedResult.Success:
+            assert self._tag is not None
+            if self._is_tag_valid(self._tag):
+                outside_text = self._outside_buffer.getvalue()
+                self._clear_buffer(self._outside_buffer)
+                self._clear_buffer(self._tag_buffer)
+                if outside_text != "":
+                    yield outside_text
+                yield self._tag
+            else:
+                self._tag.proto = self._tag_buffer.getvalue()
+                self._outside_buffer.write(self._tag.proto)
+                self._clear_buffer(self._tag_buffer)
+            self._tag = None
+            self._phase = _Phase.OUTSIDE
+
+        elif parsed_result == _ParsedResult.Failed:
+            self._outside_buffer.write(self._tag_buffer.getvalue())
+            self._clear_buffer(self._tag_buffer)
+            self._phase = _Phase.OUTSIDE
+
+    def _is_tag_valid(self, tag: Tag) -> bool:
+        if tag.kind == TagKind.CLOSING and len(tag.attributes) > 0:
+            return False
+        if tag.find_invalid_name() is not None:
+            return False
+        return True
+
+    def _clear_buffer(self, buffer: StringIO):
+        buffer.truncate(0)
+        buffer.seek(0)

epub_translator/xml/firendly/tag.py

@@ -0,0 +1,118 @@
+from collections.abc import Generator
+from dataclasses import dataclass
+from enum import Enum, auto
+from io import StringIO
+
+
+class TagKind(Enum):
+    OPENING = auto()
+    CLOSING = auto()
+    SELF_CLOSING = auto()
+
+
+@dataclass
+class Tag:
+    kind: TagKind
+    name: str
+    proto: str
+    attributes: list[tuple[str, str]]
+
+    def __str__(self):
+        buffer = StringIO()
+        buffer.write("<")
+        if self.kind == TagKind.CLOSING:
+            buffer.write("/")
+        buffer.write(self.name)
+        if len(self.attributes) > 0:
+            buffer.write(" ")
+            for i, (attr_name, attr_value) in enumerate(self.attributes):
+                buffer.write(attr_name)
+                buffer.write("=")
+                buffer.write('"')
+                buffer.write(attr_value)
+                buffer.write('"')
+                if i < len(self.attributes) - 1:
+                    buffer.write(" ")
+        if self.kind == TagKind.SELF_CLOSING:
+            buffer.write("/>")
+        else:
+            buffer.write(">")
+        return buffer.getvalue()
+
+    def find_invalid_name(self) -> str | None:
+        for name in self._iter_tag_names():
+            if not all(is_valid_value_char(c) for c in name):
+                return name
+            # https://www.w3schools.com/xml/xml_elements.asp
+            # The following logic enforces a subset of XML naming rules:
+            # - Names must not be empty.
+            # - Names must start with a letter (a-z, A-Z) or an underscore (_).
+            if name == "":
+                return name
+            char = name[0]
+            if char == "_":
+                continue
+            if "a" <= char <= "z" or "A" <= char <= "Z":
+                continue
+            return name
+
+        return None
+
+    def find_invalid_attr_value(self) -> tuple[str, str] | None:
+        for attr_name, attr_value in self.attributes:
+            if not all(is_valid_value_char(c) for c in attr_value):
+                return attr_name, attr_value
+        return None
+
+    def _iter_tag_names(self) -> Generator[str, None, None]:
+        yield self.name
+        for attr_name, _ in self.attributes:
+            yield attr_name
+
+
+# XML Attribute Values: https://www.w3.org/TR/xml/#NT-AttValue
+# URI Syntax: https://www.rfc-editor.org/rfc/rfc3986
+# HTML Attributes: https://html.spec.whatwg.org/multipage/syntax.html#attributes-2
+_VALID_VALUE_CHARS = frozenset(
+    (
+        ",",
+        ".",
+        "/",
+        "#",
+        "?",
+        "&",
+        "=",
+        ":",
+        "%",
+        ";",
+        " ",
+    )
+)
+
+
+# XML Names: https://www.w3.org/TR/xml/#NT-Name
+# XML Namespaces: https://www.w3.org/TR/xml-names/#ns-qualnames
+# HTML Custom Data Attributes: https://html.spec.whatwg.org/multipage/dom.html#custom-data-attribute
+_VALID_NAME_CHARS = frozenset(("-", "_", ":", "."))
+
+
+def is_valid_value_char(char: str) -> bool:
+    if is_valid_name_char(char):
+        return True
+    if char in _VALID_VALUE_CHARS:
+        return True
+    return False
+
+
+def is_valid_name_char(char: str) -> bool:
+    if char in _VALID_NAME_CHARS:
+        return True
+
+    # https://www.w3.org/TR/xml/#NT-Name
+    if "a" <= char <= "z":
+        return True
+    if "A" <= char <= "Z":
+        return True
+    if "0" <= char <= "9":
+        return True
+    return False

epub_translator/xml/firendly/transform.py

@@ -0,0 +1,36 @@
+from xml.etree.ElementTree import Element
+
+from .tag import Tag, TagKind
+
+
+def tag_to_element(tag: Tag) -> Element:
+    element = Element(tag.name)
+    for attr_name, attr_value in tag.attributes:
+        element.set(attr_name, attr_value)
+    return element
+
+
+def element_to_tag(element: Element, kind: TagKind, proto: str = "") -> Tag:
+    tag = Tag(
+        kind=kind,
+        name=element.tag,
+        proto=proto,
+        attributes=[],
+    )
+    if kind != TagKind.CLOSING:
+        for attr_name in sorted(list(element.keys())):
+            attr_value = element.get(attr_name, "")
+            tag.attributes.append((attr_name, attr_value))
+
+    # To make LLM easier to understand, the naming here is restricted in a more strict way.
+    # https://github.com/oomol-lab/pdf-craft/issues/149
+    invalid_name = tag.find_invalid_name()
+    if invalid_name is not None:
+        raise ValueError(f"find invalid tag name or attribute name: {invalid_name}")
+
+    invalid_attr_pair = tag.find_invalid_attr_value()
+    if invalid_attr_pair is not None:
+        attr_name, attr_value = invalid_attr_pair
+        raise ValueError(f'find invalid attribute value: {attr_name}="{attr_value}"')
+
+    return tag

epub_translator/xml/xml.py

@@ -0,0 +1,52 @@
+from collections.abc import Generator
+from xml.etree.ElementTree import Element
+
+
+def find_first(element: Element, tag: str) -> Element | None:
+    if element.tag == tag:
+        return element
+    for child in element:
+        result = find_first(child, tag)
+        if result is not None:
+            return result
+    return None
+
+
+def iter_with_stack(element: Element) -> Generator[tuple[list[Element], Element], None, None]:
+    """先序遍历：yield parent_path, element"""
+    stack: list[list[Element]] = [[element]]
+    while stack:
+        current_path = stack.pop()
+        current = current_path[-1]
+        yield current_path[:-1], current
+
+        if len(current) == 0:
+            continue
+
+        for child in reversed(list(current)):
+            child_path = list(current_path)
+            child_path.append(child)
+            stack.append(child_path)
+
+
+def clone_element(element: Element) -> Element:
+    new_element = Element(element.tag, element.attrib)
+    new_element.text = element.text
+    for child in element:
+        new_child = clone_element(child)
+        new_child.tail = child.tail
+        new_element.append(new_child)
+    return new_element
+
+
+def plain_text(element: Element) -> str:
+    return "".join(_iter_text_in(element))
+
+
+def _iter_text_in(element: Element) -> Generator[str, None, None]:
+    if element.text:
+        yield element.text
+    for child in element:
+        yield from _iter_text_in(child)
+        if child.tail:
+            yield child.tail

epub_translator/xml/xml_like.py

@@ -0,0 +1,231 @@
+import io
+import re
+import warnings
+from typing import IO
+from xml.etree.ElementTree import Element, fromstring, tostring
+
+from .xml import iter_with_stack
+
+_XML_NAMESPACE_URI = "http://www.w3.org/XML/1998/namespace"
+
+_COMMON_NAMESPACES = {
+    "http://www.w3.org/1999/xhtml": "xhtml",
+    "http://www.idpf.org/2007/ops": "epub",
+    "http://www.w3.org/1998/Math/MathML": "m",
+    "http://purl.org/dc/elements/1.1/": "dc",
+    "http://www.daisy.org/z3986/2005/ncx/": "ncx",
+    "http://www.idpf.org/2007/opf": "opf",
+    "http://www.w3.org/2000/svg": "svg",
+    "urn:oasis:names:tc:opendocument:xmlns:container": "container",
+    "http://www.w3.org/XML/1998/namespace": "xml",  # Reserved XML namespace
+}
+
+_ROOT_NAMESPACES = {
+    "http://www.w3.org/1999/xhtml",  # XHTML
+    "http://www.daisy.org/z3986/2005/ncx/",  # NCX
+    "http://www.idpf.org/2007/opf",  # OPF
+    "urn:oasis:names:tc:opendocument:xmlns:container",  # Container
+}
+
+_ENCODING_PATTERN = re.compile(r'encoding\s*=\s*["\']([^"\']+)["\']', re.IGNORECASE)
+_FIRST_ELEMENT_PATTERN = re.compile(r"<(?![?!])[a-zA-Z]")
+_NAMESPACE_IN_TAG = re.compile(r"\{([^}]+)\}")
+
+# Some non-standard EPUB generators use HTML-style tags without self-closing syntax
+# We need to convert them to XML-compatible format before parsing
+_EMPTY_TAGS = (
+    "br",
+    "hr",
+    "input",
+    "col",
+    "base",
+    "meta",
+    "area",
+)
+
+# For reading: match tags like <br> or <br class="x"> (but not <br/> or <body>)
+_EMPTY_TAG_OPEN_PATTERN = re.compile(r"<(" + "|".join(_EMPTY_TAGS) + r")(\s[^/>]*)>")
+
+# For saving: match self-closing tags like <br />
+_EMPTY_TAG_CLOSE_PATTERN = re.compile(r"<(" + "|".join(_EMPTY_TAGS) + r")(\s[^>]*?)\s*/>")
+
+
+class XMLLikeNode:
+    def __init__(self, file: IO[bytes], is_html_like: bool = False) -> None:
+        raw_content = file.read()
+        self._encoding: str = self._detect_encoding(raw_content)
+        content = raw_content.decode(self._encoding)
+        self._header, xml_content = self._extract_header(content)
+        self._namespaces: dict[str, str] = {}
+        self._tag_to_namespace: dict[str, str] = {}
+        self._attr_to_namespace: dict[str, str] = {}
+
+        # For non-standard HTML files, convert <br> to <br/> before parsing
+        self._is_html_like = is_html_like
+        if is_html_like:
+            xml_content = re.sub(
+                pattern=_EMPTY_TAG_OPEN_PATTERN,
+                repl=lambda m: f"<{m.group(1)}{m.group(2)} />",
+                string=xml_content,
+            )
+
+        try:
+            self.element = self._extract_and_clean_namespaces(
+                element=fromstring(xml_content),
+            )
+        except Exception as error:
+            raise ValueError("Failed to parse XML-like content") from error
+
+    @property
+    def encoding(self) -> str:
+        return self._encoding
+
+    @property
+    def namespaces(self) -> list[str]:
+        return list(self._namespaces.keys())
+
+    def save(self, file: IO[bytes]) -> None:
+        writer = io.TextIOWrapper(file, encoding=self._encoding, write_through=True)
+        try:
+            if self._header:
+                writer.write(self._header)
+
+            content = self._serialize_with_namespaces(self.element)
+
+            # For non-standard HTML files, convert back from <br/> to <br>
+            if self._is_html_like:
+                content = re.sub(
+                    pattern=_EMPTY_TAG_CLOSE_PATTERN,
+                    repl=lambda m: f"<{m.group(1)}{m.group(2)}>",
+                    string=content,
+                )
+
+            writer.write(content)
+
+        finally:
+            writer.detach()
+
+    def _detect_encoding(self, raw_content: bytes) -> str:
+        if raw_content.startswith(b"\xef\xbb\xbf"):
+            return "utf-8-sig"
+        elif raw_content.startswith(b"\xff\xfe"):
+            return "utf-16-le"
+        elif raw_content.startswith(b"\xfe\xff"):
+            return "utf-16-be"
+
+        # 尝试从 XML 声明中提取编码：只读取前 1024 字节来查找 XML 声明
+        header_bytes = raw_content[:1024]
+        for try_encoding in ("utf-8", "utf-16-le", "utf-16-be", "iso-8859-1"):
+            try:
+                header_str = header_bytes.decode(try_encoding)
+                match = _ENCODING_PATTERN.search(header_str)
+                if match:
+                    declared_encoding = match.group(1).lower()
+                    try:
+                        raw_content.decode(declared_encoding)
+                        return declared_encoding
+                    except (LookupError, UnicodeDecodeError):
+                        pass
+            except UnicodeDecodeError:
+                continue
+
+        try:
+            raw_content.decode("utf-8")
+            return "utf-8"
+        except UnicodeDecodeError:
+            pass
+        return "iso-8859-1"
+
+    def _extract_header(self, content: str) -> tuple[str, str]:
+        match = _FIRST_ELEMENT_PATTERN.search(content)
+        if match:
+            split_pos = match.start()
+            header = content[:split_pos]
+            xml_content = content[split_pos:]
+            return header, xml_content
+        return "", content
+
+    def _extract_and_clean_namespaces(self, element: Element) -> Element:
+        for _, elem in iter_with_stack(element):
+            match = _NAMESPACE_IN_TAG.match(elem.tag)
+            if match:
+                namespace_uri = match.group(1)
+                if namespace_uri not in self._namespaces:
+                    prefix = _COMMON_NAMESPACES.get(namespace_uri, f"ns{len(self._namespaces)}")
+                    self._namespaces[namespace_uri] = prefix
+
+                tag_name = elem.tag[len(match.group(0)) :]
+
+                # Record tag -> namespace mapping (warn if conflict)
+                if tag_name in self._tag_to_namespace and self._tag_to_namespace[tag_name] != namespace_uri:
+                    warnings.warn(
+                        f"Tag '{tag_name}' has multiple namespaces: "
+                        f"{self._tag_to_namespace[tag_name]} and {namespace_uri}. "
+                        f"Using the first one.",
+                        stacklevel=2,
+                    )
+                else:
+                    self._tag_to_namespace[tag_name] = namespace_uri
+
+                # Clean: remove namespace URI completely
+                elem.tag = tag_name
+
+            for attr_key in list(elem.attrib.keys()):
+                match = _NAMESPACE_IN_TAG.match(attr_key)
+                if match:
+                    namespace_uri = match.group(1)
+                    if namespace_uri not in self._namespaces:
+                        prefix = _COMMON_NAMESPACES.get(namespace_uri, f"ns{len(self._namespaces)}")
+                        self._namespaces[namespace_uri] = prefix
+
+                    attr_name = attr_key[len(match.group(0)) :]
+                    attr_value = elem.attrib.pop(attr_key)
+
+                    # Record attr -> namespace mapping (warn if conflict)
+                    if attr_name in self._attr_to_namespace and self._attr_to_namespace[attr_name] != namespace_uri:
+                        warnings.warn(
+                            f"Attribute '{attr_name}' has multiple namespaces: "
+                            f"{self._attr_to_namespace[attr_name]} and {namespace_uri}. "
+                            f"Using the first one.",
+                            stacklevel=2,
+                        )
+                    else:
+                        self._attr_to_namespace[attr_name] = namespace_uri
+
+                    # Clean: remove namespace URI completely
+                    elem.attrib[attr_name] = attr_value
+        return element
+
+    def _serialize_with_namespaces(self, element: Element) -> str:
+        # First, add namespace declarations to root element (before serialization)
+        for namespace_uri, prefix in self._namespaces.items():
+            # Skip the reserved xml namespace - it's implicit
+            if namespace_uri == _XML_NAMESPACE_URI:
+                continue
+            if namespace_uri in _ROOT_NAMESPACES:
+                element.attrib["xmlns"] = namespace_uri
+            else:
+                element.attrib[f"xmlns:{prefix}"] = namespace_uri
+
+        # Serialize the element tree as-is (tags are simple names without prefixes)
+        xml_string = tostring(element, encoding="unicode")
+
+        # Now restore namespace prefixes in the serialized string
+        # For each tag that should have a namespace prefix, wrap it with the prefix
+        for tag_name, namespace_uri in self._tag_to_namespace.items():
+            if namespace_uri not in _ROOT_NAMESPACES:
+                # Get the prefix for this namespace
+                prefix = self._namespaces[namespace_uri]
+                # Replace opening and closing tags
+                xml_string = xml_string.replace(f"<{tag_name} ", f"<{prefix}:{tag_name} ")
+                xml_string = xml_string.replace(f"<{tag_name}>", f"<{prefix}:{tag_name}>")
+                xml_string = xml_string.replace(f"</{tag_name}>", f"</{prefix}:{tag_name}>")
+                xml_string = xml_string.replace(f"<{tag_name}/>", f"<{prefix}:{tag_name}/>")
+
+        # Similarly for attributes (though less common in EPUB)
+        for attr_name, namespace_uri in self._attr_to_namespace.items():
+            if namespace_uri not in _ROOT_NAMESPACES:
+                prefix = self._namespaces[namespace_uri]
+                xml_string = xml_string.replace(f' {attr_name}="', f' {prefix}:{attr_name}="')
+
+        return xml_string

epub_translator/xml_translator/__init__.py

@@ -0,0 +1,3 @@
+from .group import XMLGroupContext
+from .submitter import submit_text_segments
+from .translator import XMLTranslator

epub_translator/xml_translator/const.py

@@ -0,0 +1,2 @@
+ID_KEY: str = "id"
+DATA_ORIGIN_LEN_KEY = "data-orig-len"