epub-translator 0.1.1__py3-none-any.whl → 0.1.4__py3-none-any.whl

epub_translator/translator.py

@@ -1,214 +1,182 @@
-from collections.abc import Callable
+from collections.abc import Callable, Generator
+from dataclasses import dataclass
+from enum import Enum, auto
+from importlib.metadata import version as get_package_version
+from os import PathLike
 from pathlib import Path
-from xml.etree.ElementTree import Element
 
-from .epub import Placeholder, Zip, is_placeholder_tag, read_toc, search_spine_paths, write_toc
-from .epub.common import find_opf_path
+from .epub import (
+    Zip,
+    read_metadata,
+    read_toc,
+    search_spine_paths,
+    write_metadata,
+    write_toc,
+)
+from .epub_transcode import decode_metadata, decode_toc_list, encode_metadata, encode_toc_list
 from .llm import LLM
-from .xml import XMLLikeNode, deduplicate_ids_in_element, find_first, plain_text
-from .xml_translator import XMLGroupContext, XMLTranslator, submit_text_segments
+from .punctuation import unwrap_french_quotes
+from .xml import XMLLikeNode, deduplicate_ids_in_element, find_first
+from .xml_interrupter import XMLInterrupter
+from .xml_translator import FillFailedEvent, SubmitKind, TranslationTask, XMLTranslator
+
+
+class _ElementType(Enum):
+    TOC = auto()
+    METADATA = auto()
+    CHAPTER = auto()
+
+
+@dataclass
+class _ElementContext:
+    element_type: _ElementType
+    chapter_data: tuple[Path, XMLLikeNode] | None = None
 
 
 def translate(
-    llm: LLM,
-    source_path: Path,
-    target_path: Path,
+    source_path: PathLike | str,
+    target_path: PathLike | str,
     target_language: str,
+    submit: SubmitKind,
     user_prompt: str | None = None,
     max_retries: int = 5,
     max_group_tokens: int = 1200,
+    llm: LLM | None = None,
+    translation_llm: LLM | None = None,
+    fill_llm: LLM | None = None,
     on_progress: Callable[[float], None] | None = None,
+    on_fill_failed: Callable[[FillFailedEvent], None] | None = None,
 ) -> None:
+    translation_llm = translation_llm or llm
+    fill_llm = fill_llm or llm
+    if translation_llm is None:
+        raise ValueError("Either translation_llm or llm must be provided")
+    if fill_llm is None:
+        raise ValueError("Either fill_llm or llm must be provided")
+
     translator = XMLTranslator(
-        llm=llm,
+        translation_llm=translation_llm,
+        fill_llm=fill_llm,
         target_language=target_language,
         user_prompt=user_prompt,
         ignore_translated_error=False,
         max_retries=max_retries,
         max_fill_displaying_errors=10,
-        group_context=XMLGroupContext(
-            encoding=llm.encoding,
-            max_group_tokens=max_group_tokens,
-        ),
+        max_group_tokens=max_group_tokens,
+        cache_seed_content=f"{_get_version()}:{target_language}",
     )
-    with Zip(source_path, target_path) as zip:
-        # Progress distribution: TOC 3%, metadata 2%, chapters 95%
-        TOC_PROGRESS = 0.03
-        METADATA_PROGRESS = 0.02
-        CHAPTERS_PROGRESS = 0.95
-
-        # Count total chapters for progress calculation (lightweight, no content loading)
-        total_chapters = _count_chapters(zip)
-        chapter_progress_step = CHAPTERS_PROGRESS / total_chapters if total_chapters > 0 else 0
-
+    with Zip(
+        source_path=Path(source_path).resolve(),
+        target_path=Path(target_path).resolve(),
+    ) as zip:
+        # mimetype should be the first file in the EPUB ZIP
+        zip.migrate(Path("mimetype"))
+
+        total_chapters = sum(1 for _, _ in search_spine_paths(zip))
+        toc_list = read_toc(zip)
+        metadata_fields = read_metadata(zip)
+
+        # Calculate weights: TOC (5%), Metadata (5%), Chapters (90%)
+        toc_has_items = len(toc_list) > 0
+        metadata_has_items = len(metadata_fields) > 0
+        total_items = (1 if toc_has_items else 0) + (1 if metadata_has_items else 0) + total_chapters
+
+        if total_items == 0:
+            return
+
+        interrupter = XMLInterrupter()
+        toc_weight = 0.05 if toc_has_items else 0
+        metadata_weight = 0.05 if metadata_has_items else 0
+        chapters_weight = 1.0 - toc_weight - metadata_weight
+        progress_per_chapter = chapters_weight / total_chapters if total_chapters > 0 else 0
         current_progress = 0.0
 
-        # Translate TOC
-        _translate_toc(translator, zip)
-        current_progress += TOC_PROGRESS
-        if on_progress:
-            on_progress(current_progress)
-
-        # Translate metadata
-        _translate_metadata(translator, zip)
-        current_progress += METADATA_PROGRESS
-        if on_progress:
-            on_progress(current_progress)
-
-        # Translate chapters
-        processed_chapters = 0
-        for element, text_segments, (chapter_path, xml, placeholder) in translator.translate_to_text_segments(
-            items=_search_chapter_items(zip),
+        for translated_elem, context in translator.translate_elements(
+            interrupt_source_text_segments=interrupter.interrupt_source_text_segments,
+            interrupt_translated_text_segments=interrupter.interrupt_translated_text_segments,
+            interrupt_block_element=interrupter.interrupt_block_element,
+            on_fill_failed=on_fill_failed,
+            tasks=_generate_tasks_from_book(
+                zip=zip,
+                toc_list=toc_list,
+                metadata_fields=metadata_fields,
+                submit=submit,
+            ),
         ):
-            submit_text_segments(
-                element=element,
-                text_segments=(
-                    segment
-                    for segment in text_segments
-                    if not any(is_placeholder_tag(e.tag) for e in segment.parent_stack)
-                ),
-            )
-            placeholder.recover()
-            deduplicate_ids_in_element(xml.element)
-            with zip.replace(chapter_path) as target_file:
-                xml.save(target_file)
-
-            # Update progress after each chapter
-            processed_chapters += 1
-            current_progress = TOC_PROGRESS + METADATA_PROGRESS + (processed_chapters * chapter_progress_step)
-            if on_progress:
-                on_progress(current_progress)
-
-
-def _translate_toc(translator: XMLTranslator, zip: Zip):
-    """Translate TOC (Table of Contents) titles."""
-    toc_list = read_toc(zip)
-    if not toc_list:
-        return
-
-    # Collect all titles recursively
-    titles_to_translate: list[str] = []
-
-    def collect_titles(items):
-        for item in items:
-            titles_to_translate.append(item.title)
-            if item.children:
-                collect_titles(item.children)
-
-    collect_titles(toc_list)
-
-    # Create XML elements for translation
-    elements_to_translate = Element("toc")
-    elements_to_translate.extend(_create_text_element(title) for title in titles_to_translate)
-
-    # Translate all titles at once
-    translated_element = translator.translate_to_element(elements_to_translate)
-
-    # Extract translated texts
-    from builtins import zip as builtin_zip
-
-    translated_titles = [
-        plain_text(elem) if elem is not None else original
-        for elem, original in builtin_zip(translated_element, titles_to_translate)
-    ]
-
-    # Fill back translated titles
-    title_index = 0
-
-    def fill_titles(items):
-        nonlocal title_index
-        for item in items:
-            item.title = translated_titles[title_index]
-            title_index += 1
-            if item.children:
-                fill_titles(item.children)
-
-    fill_titles(toc_list)
-
-    # Write back the translated TOC
-    write_toc(zip, toc_list)
-
-
-def _translate_metadata(translator: XMLTranslator, zip: Zip):
-    """Translate metadata fields in OPF file."""
-    opf_path = find_opf_path(zip)
-
-    with zip.read(opf_path) as f:
-        xml = XMLLikeNode(f)
-
-    # Find metadata element
-    metadata_elem = None
-    for child in xml.element:
-        if child.tag.endswith("metadata"):
-            metadata_elem = child
-            break
-
-    if metadata_elem is None:
-        return
-
-    # Collect metadata fields to translate
-    # Skip fields that should not be translated
-    skip_fields = {
-        "language",
-        "identifier",
-        "date",
-        "meta",
-        "contributor",  # Usually technical information
-    }
-
-    fields_to_translate: list[tuple[Element, str]] = []
-
-    for elem in metadata_elem:
-        # Get tag name without namespace
-        tag_name = elem.tag.split("}")[-1] if "}" in elem.tag else elem.tag
-
-        # Check if element has text content and should be translated
-        if elem.text and elem.text.strip() and tag_name not in skip_fields:
-            fields_to_translate.append((elem, elem.text.strip()))
-
-    if not fields_to_translate:
-        return
-
-    # Create XML elements for translation
-    elements_to_translate = Element("metadata")
-    elements_to_translate.extend(_create_text_element(text) for _, text in fields_to_translate)
-
-    # Translate all metadata at once
-    translated_element = translator.translate_to_element(elements_to_translate)
-
-    # Fill back translated texts
-    from builtins import zip as builtin_zip
-
-    for (elem, _), translated_elem in builtin_zip(fields_to_translate, translated_element, strict=True):
-        if translated_elem is not None:
-            translated_text = plain_text(translated_elem)
-            if translated_text:
-                elem.text = translated_text
-
-    # Write back the modified OPF file
-    with zip.replace(opf_path) as f:
-        xml.save(f)
-
-
-def _count_chapters(zip: Zip) -> int:
-    """Count total chapters without loading content (lightweight)."""
-    return sum(1 for _ in search_spine_paths(zip))
-
-
-def _search_chapter_items(zip: Zip):
-    for chapter_path in search_spine_paths(zip):
+            if context.element_type == _ElementType.TOC:
+                translated_elem = unwrap_french_quotes(translated_elem)
+                decoded_toc = decode_toc_list(translated_elem)
+                write_toc(zip, decoded_toc)
+
+                current_progress += toc_weight
+                if on_progress:
+                    on_progress(current_progress)
+
+            elif context.element_type == _ElementType.METADATA:
+                translated_elem = unwrap_french_quotes(translated_elem)
+                decoded_metadata = decode_metadata(translated_elem)
+                write_metadata(zip, decoded_metadata)
+
+                current_progress += metadata_weight
+                if on_progress:
+                    on_progress(current_progress)
+
+            elif context.element_type == _ElementType.CHAPTER:
+                if context.chapter_data is not None:
+                    chapter_path, xml = context.chapter_data
+                    deduplicate_ids_in_element(xml.element)
+                    with zip.replace(chapter_path) as target_file:
+                        xml.save(target_file)
+
+                current_progress += progress_per_chapter
+                if on_progress:
+                    on_progress(current_progress)
+
+
+def _generate_tasks_from_book(
+    zip: Zip,
+    toc_list: list,
+    metadata_fields: list,
+    submit: SubmitKind,
+) -> Generator[TranslationTask[_ElementContext], None, None]:
+    head_submit = submit
+    if head_submit == SubmitKind.APPEND_BLOCK:
+        head_submit = SubmitKind.APPEND_TEXT
+
+    if toc_list:
+        yield TranslationTask(
+            element=encode_toc_list(toc_list),
+            action=head_submit,
+            payload=_ElementContext(element_type=_ElementType.TOC),
+        )
+
+    if metadata_fields:
+        yield TranslationTask(
+            element=encode_metadata(metadata_fields),
+            action=head_submit,
+            payload=_ElementContext(element_type=_ElementType.METADATA),
+        )
+
+    for chapter_path, media_type in search_spine_paths(zip):
         with zip.read(chapter_path) as chapter_file:
             xml = XMLLikeNode(
                 file=chapter_file,
-                is_html_like=chapter_path.suffix.lower() in (".html", ".htm"),
+                is_html_like=(media_type == "text/html"),
             )
         body_element = find_first(xml.element, "body")
         if body_element is not None:
-            placeholder = Placeholder(body_element)
-            yield body_element, (chapter_path, xml, placeholder)
+            yield TranslationTask(
+                element=body_element,
+                action=submit,
+                payload=_ElementContext(
+                    element_type=_ElementType.CHAPTER,
+                    chapter_data=(chapter_path, xml),
+                ),
+            )
 
 
-def _create_text_element(text: str) -> Element:
-    elem = Element("text")
-    elem.text = text
-    return elem
+def _get_version() -> str:
+    try:
+        return get_package_version("epub-translator")
+    except Exception:
+        return "development"

epub_translator/utils.py

@@ -1,7 +1,40 @@
 import re
+from collections.abc import Iterable
+from typing import TypeVar
+
+K = TypeVar("K")
+T = TypeVar("T")
 
 _WHITESPACE_PATTERN = re.compile(r"\s+")
 
 
 def normalize_whitespace(text: str) -> str:
     return _WHITESPACE_PATTERN.sub(" ", text)
+
+
+def is_the_same(elements: Iterable[T]) -> bool:
+    iterator = iter(elements)
+    try:
+        first_element = next(iterator)
+    except StopIteration:
+        return True
+
+    for element in iterator:
+        if element != first_element:
+            return False
+    return True
+
+
+def nest(items: Iterable[tuple[K, T]]) -> dict[K, list[T]]:
+    nested_dict: dict[K, list[T]] = {}
+    for key, value in items:
+        ensure_list(nested_dict, key).append(value)
+    return nested_dict
+
+
+def ensure_list(target: dict[K, list[T]], key: K) -> list[T]:
+    value = target.get(key, None)
+    if value is None:
+        value = []
+        target[key] = value
+    return value

epub_translator/xml/__init__.py

@@ -1,4 +1,7 @@
+from .const import *
 from .deduplication import *
 from .firendly import *
+from .inline import *
+from .utils import *
 from .xml import *
 from .xml_like import *

epub_translator/xml/const.py

@@ -0,0 +1 @@
+ID_KEY: str = "id"

epub_translator/xml/deduplication.py

@@ -1,8 +1,8 @@
 from xml.etree.ElementTree import Element
 
+from .const import ID_KEY
 from .xml import iter_with_stack
 
-_ID_KEY = "id"
 _SUFFIX = "__translated"
 
 
@@ -11,9 +11,9 @@ def deduplicate_ids_in_element(element: Element) -> Element:
     original_id_count: dict[str, int] = {}
 
     for _, sub_element in iter_with_stack(element):
-        if _ID_KEY not in sub_element.attrib:
+        if ID_KEY not in sub_element.attrib:
             continue
-        original_id = sub_element.attrib[_ID_KEY]
+        original_id = sub_element.attrib[ID_KEY]
 
         if original_id not in seen_ids:
             seen_ids.add(original_id)

epub_translator/xml/inline.py

@@ -0,0 +1,67 @@
+# HTML inline-level elements
+# Reference: https://developer.mozilla.org/en-US/docs/Web/HTML/Inline_elements
+# Reference: https://developer.mozilla.org/en-US/docs/Glossary/Inline-level_content
+_HTML_INLINE_TAGS = frozenset(
+    (
+        # Inline text semantics
+        "a",
+        "abbr",
+        "b",
+        "bdi",
+        "bdo",
+        "br",
+        "cite",
+        "code",
+        "data",
+        "dfn",
+        "em",
+        "i",
+        "kbd",
+        "mark",
+        "q",
+        "rp",
+        "rt",
+        "ruby",
+        "s",
+        "samp",
+        "small",
+        "span",
+        "strong",
+        "sub",
+        "sup",
+        "time",
+        "u",
+        "var",
+        "wbr",
+        # Image and multimedia
+        "img",
+        "svg",
+        "canvas",
+        "audio",
+        "video",
+        "map",
+        "area",
+        # Form elements
+        "input",
+        "button",
+        "select",
+        "textarea",
+        "label",
+        "output",
+        "progress",
+        "meter",
+        # Embedded content
+        "iframe",
+        "embed",
+        "object",
+        # Other inline elements
+        "script",
+        "del",
+        "ins",
+        "slot",
+    )
+)
+
+
+def is_inline_tag(tag: str) -> bool:
+    return tag.lower() in _HTML_INLINE_TAGS