epub-blocks 0.2.0__py3-none-any.whl

epub_blocks/__init__.py

@@ -0,0 +1,55 @@
+"""Public API for deterministic, recipe-driven EPUB text extraction."""
+
+from importlib.metadata import version
+
+from .errors import EpubBlocksError
+from .extract import extract_blocks, extract_fragments
+from .models import (
+    BlockReference,
+    CompiledBlock,
+    CompiledRecipe,
+    EpubPackage,
+    ExtractedBlock,
+    Fragment,
+    NormalizationOptions,
+    SpineDocument,
+    TextBlock,
+)
+from .package import inspect_epub
+from .recipe import (
+    compile_recipe,
+    compile_recipe_file,
+    compiled_recipe_digest,
+    extract_recipe,
+    extract_recipe_file,
+    load_recipe,
+    write_tsv,
+)
+from .safety import SafetyLimits
+
+__version__ = version("epub-blocks")
+
+__all__ = [
+    "BlockReference",
+    "CompiledBlock",
+    "CompiledRecipe",
+    "EpubBlocksError",
+    "EpubPackage",
+    "ExtractedBlock",
+    "Fragment",
+    "NormalizationOptions",
+    "SafetyLimits",
+    "SpineDocument",
+    "TextBlock",
+    "__version__",
+    "compile_recipe",
+    "compile_recipe_file",
+    "compiled_recipe_digest",
+    "extract_blocks",
+    "extract_fragments",
+    "extract_recipe",
+    "extract_recipe_file",
+    "inspect_epub",
+    "load_recipe",
+    "write_tsv",
+]

epub_blocks/cli.py

@@ -0,0 +1,37 @@
+from __future__ import annotations
+
+import argparse
+import sys
+from pathlib import Path
+
+from . import __version__
+from .errors import EpubBlocksError
+from .recipe import extract_recipe_file, write_tsv
+
+
+def main() -> int:
+    """Run the command-line extractor and return a process exit status."""
+
+    parser = argparse.ArgumentParser(
+        description="Apply a recipe to an EPUB and write structured text as TSV."
+    )
+    parser.add_argument("epub", type=Path)
+    parser.add_argument("recipe", type=Path)
+    parser.add_argument("output", type=Path)
+    parser.add_argument(
+        "--version", action="version", version=f"%(prog)s {__version__}"
+    )
+    args = parser.parse_args()
+
+    try:
+        blocks = extract_recipe_file(args.epub, args.recipe)
+        write_tsv(args.output, blocks)
+    except (EpubBlocksError, OSError) as error:
+        print(f"epub-blocks: error: {error}", file=sys.stderr)
+        return 2
+    print(f"{len(blocks)} blocks -> {args.output}")
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

epub_blocks/errors.py

@@ -0,0 +1,2 @@
+class EpubBlocksError(ValueError):
+    """Raised when an EPUB cannot be inspected or extracted as requested."""

epub_blocks/extract.py

@@ -0,0 +1,237 @@
+from __future__ import annotations
+
+import copy
+import hashlib
+import os
+import re
+from collections.abc import Iterator, Sequence
+from pathlib import Path
+from typing import BinaryIO
+from zipfile import BadZipFile, ZipFile
+
+from .errors import EpubBlocksError
+from .models import EpubPackage, Fragment, NormalizationOptions, TextBlock
+from .package import matches, read_epub_package, select_spine_documents
+from .safety import DEFAULT_SAFETY_LIMITS, EpubArchive, SafetyLimits
+from .xhtml import (
+    extract_fragment,
+    flatten_text,
+    normalize_text,
+    read_document_body,
+    remove_descendants_by_epub_type,
+)
+from .xml import XmlElement, local_name
+
+PRIMARY_BLOCK_TAGS = frozenset({"p", "h1", "h2", "h3", "h4", "h5", "h6", "pre"})
+FALLBACK_BLOCK_TAGS = frozenset({"blockquote", "li"})
+DEFAULT_OMITTED_EPUB_TYPES = frozenset({"noteref", "pagebreak"})
+_DEFAULT_NORMALIZATION = NormalizationOptions()
+_SHA256 = re.compile(r"[0-9a-f]{64}")
+StrPath = str | os.PathLike[str]
+
+
+def _stream_sha256(source: BinaryIO) -> str:
+    digest = hashlib.sha256()
+    while block := source.read(1024 * 1024):
+        digest.update(block)
+    return digest.hexdigest()
+
+
+def _contains_primary_block(element: XmlElement, primary_tags: frozenset[str]) -> bool:
+    return any(
+        descendant is not element and local_name(descendant.tag) in primary_tags
+        for descendant in element.iter()
+    )
+
+
+def _candidate_elements(
+    parent: XmlElement,
+    primary_tags: frozenset[str],
+    fallback_tags: frozenset[str],
+    parent_path: tuple[int, ...] = (),
+) -> Iterator[tuple[XmlElement, tuple[int, ...]]]:
+    for index, child in enumerate(list(parent), 1):
+        element_path = (*parent_path, index)
+        tag = local_name(child.tag)
+        if tag in primary_tags or (
+            tag in fallback_tags and not _contains_primary_block(child, primary_tags)
+        ):
+            yield child, element_path
+        else:
+            yield from _candidate_elements(
+                child, primary_tags, fallback_tags, element_path
+            )
+
+
+def _extract_blocks_from_epub(
+    epub: EpubArchive,
+    package: EpubPackage,
+    source_path: Path,
+    *,
+    include_documents: Sequence[str] | None = None,
+    exclude_documents: Sequence[str] | None = None,
+    exclude_classes: Sequence[str] | None = None,
+    include_locators: Sequence[str] | None = None,
+    exclude_locators: Sequence[str] | None = None,
+    include_non_linear: bool = False,
+    primary_tags: frozenset[str] = PRIMARY_BLOCK_TAGS,
+    fallback_tags: frozenset[str] = FALLBACK_BLOCK_TAGS,
+    omit_epub_types: frozenset[str] = DEFAULT_OMITTED_EPUB_TYPES,
+    normalization: NormalizationOptions = _DEFAULT_NORMALIZATION,
+    document_cache: dict[str, XmlElement] | None = None,
+) -> list[TextBlock]:
+    """Extract blocks from an already-open, bounded EPUB archive."""
+
+    blocks: list[TextBlock] = []
+    excluded_classes = {value.casefold() for value in (exclude_classes or ())}
+    documents = select_spine_documents(
+        package,
+        include_documents,
+        exclude_documents,
+        include_non_linear=include_non_linear,
+    )
+    cache = document_cache if document_cache is not None else {}
+    for document in documents:
+        body = read_document_body(epub, document.path, cache)
+        for element, address in _candidate_elements(body, primary_tags, fallback_tags):
+            classes = frozenset(element.get("class", "").split())
+            if {value.casefold() for value in classes} & excluded_classes:
+                continue
+            element_path = ".".join(str(component) for component in address)
+            locator = f"{document.path}#{element_path}"
+            if include_locators and not matches(locator, include_locators):
+                continue
+            if exclude_locators and matches(locator, exclude_locators):
+                continue
+            selected = copy.deepcopy(element)
+            remove_descendants_by_epub_type(selected, omit_epub_types)
+            text = normalize_text(flatten_text(selected), normalization)
+            if not text:
+                continue
+            blocks.append(
+                TextBlock(
+                    spine_position=document.position,
+                    document_path=document.path,
+                    element_path=element_path,
+                    tag=local_name(element.tag),
+                    text=text,
+                    classes=classes,
+                )
+            )
+    if not blocks:
+        raise EpubBlocksError(f"{source_path}: no text blocks matched the selection")
+    return blocks
+
+
+def extract_blocks(
+    epub_path: StrPath,
+    *,
+    expected_identifier: str | None = None,
+    include_documents: Sequence[str] | None = None,
+    exclude_documents: Sequence[str] | None = None,
+    exclude_classes: Sequence[str] | None = None,
+    include_locators: Sequence[str] | None = None,
+    exclude_locators: Sequence[str] | None = None,
+    include_non_linear: bool = False,
+    primary_tags: frozenset[str] = PRIMARY_BLOCK_TAGS,
+    fallback_tags: frozenset[str] = FALLBACK_BLOCK_TAGS,
+    omit_epub_types: frozenset[str] = DEFAULT_OMITTED_EPUB_TYPES,
+    normalization: NormalizationOptions = _DEFAULT_NORMALIZATION,
+    limits: SafetyLimits = DEFAULT_SAFETY_LIMITS,
+) -> list[TextBlock]:
+    """Extract selected text blocks in EPUB spine and XHTML document order."""
+
+    path = Path(epub_path)
+    try:
+        with ZipFile(path) as zip_file:
+            epub = EpubArchive(zip_file, limits)
+            package = read_epub_package(epub)
+            if expected_identifier is not None:
+                if not expected_identifier:
+                    raise EpubBlocksError(
+                        "expected_identifier must be a non-empty string"
+                    )
+                if expected_identifier not in package.identifiers:
+                    raise EpubBlocksError(
+                        f"{path}: expected package identifier "
+                        f"{expected_identifier!r} not found"
+                    )
+            return _extract_blocks_from_epub(
+                epub,
+                package,
+                path,
+                include_documents=include_documents,
+                exclude_documents=exclude_documents,
+                exclude_classes=exclude_classes,
+                include_locators=include_locators,
+                exclude_locators=exclude_locators,
+                include_non_linear=include_non_linear,
+                primary_tags=primary_tags,
+                fallback_tags=fallback_tags,
+                omit_epub_types=omit_epub_types,
+                normalization=normalization,
+            )
+    except BadZipFile as error:
+        raise EpubBlocksError(f"{path}: not a valid ZIP container") from error
+
+
+def extract_fragments(
+    epub_path: StrPath,
+    fragments: Sequence[Fragment],
+    *,
+    expected_identifier: str | None = None,
+    expected_sha256: str | None = None,
+    normalization: NormalizationOptions = _DEFAULT_NORMALIZATION,
+    omit_epub_types: frozenset[str] = DEFAULT_OMITTED_EPUB_TYPES,
+    limits: SafetyLimits = DEFAULT_SAFETY_LIMITS,
+) -> list[str]:
+    """Extract arbitrary XHTML fragments through one validated EPUB read.
+
+    Each result corresponds to the fragment at the same position. Descendant
+    omissions precede normalization; slice offsets index normalized text.
+    Callers that need a pinned input can require both a package identifier and
+    complete-file SHA-256 without reopening the archive for every fragment.
+    """
+
+    if expected_identifier is not None and not expected_identifier:
+        raise EpubBlocksError("expected_identifier must be a non-empty string")
+    if expected_sha256 is not None and _SHA256.fullmatch(expected_sha256) is None:
+        raise EpubBlocksError(
+            "expected_sha256 must be 64 lowercase hexadecimal characters"
+        )
+
+    path = Path(epub_path)
+    with path.open("rb") as source:
+        if expected_sha256 is not None:
+            if _stream_sha256(source) != expected_sha256:
+                raise EpubBlocksError(f"{path}: source SHA-256 does not match")
+            source.seek(0)
+        try:
+            zip_file = ZipFile(source)
+        except BadZipFile as error:
+            raise EpubBlocksError(f"{path}: not a valid ZIP container") from error
+        with zip_file:
+            epub = EpubArchive(zip_file, limits)
+            package = read_epub_package(epub)
+            if (
+                expected_identifier is not None
+                and expected_identifier not in package.identifiers
+            ):
+                raise EpubBlocksError(
+                    f"{path}: expected package identifier "
+                    f"{expected_identifier!r} not found"
+                )
+            cache: dict[str, XmlElement] = {}
+            return [
+                normalize_text(
+                    extract_fragment(
+                        epub,
+                        fragment,
+                        cache=cache,
+                        normalization=normalization,
+                        omit_epub_types=omit_epub_types,
+                    ),
+                    normalization,
+                )
+                for fragment in fragments
+            ]

epub_blocks/models.py

@@ -0,0 +1,106 @@
+from __future__ import annotations
+
+from dataclasses import dataclass
+from typing import Literal
+
+UnicodeNormalization = Literal["NFC", "NFD", "NFKC", "NFKD", "none"]
+
+
+@dataclass(frozen=True)
+class SpineDocument:
+    """One XHTML document in EPUB spine order."""
+
+    position: int
+    path: str
+    properties: frozenset[str]
+    linear: bool = True
+
+
+@dataclass(frozen=True)
+class EpubPackage:
+    """The package information needed for deterministic text extraction."""
+
+    package_path: str
+    identifiers: frozenset[str]
+    spine: tuple[SpineDocument, ...]
+
+
+@dataclass(frozen=True)
+class NormalizationOptions:
+    """Text normalization applied after XHTML text extraction."""
+
+    collapse_whitespace: bool = True
+    strip: bool = True
+    unicode_normalization: UnicodeNormalization = "NFC"
+
+
+@dataclass(frozen=True)
+class Fragment:
+    """A selectable XHTML subtree, with optional omissions and text slice."""
+
+    document_path: str
+    element_path: str = ""
+    omit_paths: tuple[str, ...] = ()
+    start: int | None = None
+    end: int | None = None
+
+
+@dataclass(frozen=True)
+class BlockReference:
+    """The canonical identifier and type assigned to one output block."""
+
+    block_id: str
+    block_type: str
+
+
+@dataclass(frozen=True)
+class CompiledBlock:
+    """One output block in a compiled extraction plan."""
+
+    block_id: str
+    block_type: str
+    parts: tuple[Fragment, ...]
+    separator: str = ""
+    consumed_locators: tuple[str, ...] = ()
+
+
+@dataclass(frozen=True)
+class CompiledRecipe:
+    """The deterministic extraction plan produced from a version 1 recipe."""
+
+    epub_identifier: str
+    epub_sha256: str
+    normalization: NormalizationOptions
+    omit_epub_types: frozenset[str]
+    blocks: tuple[CompiledBlock, ...]
+    skipped_locators: tuple[str, ...] = ()
+    reserved_locators: tuple[str, ...] = ()
+
+
+@dataclass(frozen=True)
+class TextBlock:
+    """An extracted XHTML block with a stable source locator."""
+
+    spine_position: int
+    document_path: str
+    element_path: str
+    tag: str
+    text: str
+    classes: frozenset[str]
+
+    @property
+    def locator(self) -> str:
+        return f"{self.document_path}#{self.element_path}"
+
+    @property
+    def source_locator(self) -> str:
+        return f"s{self.spine_position:03d}:{self.locator}"
+
+
+@dataclass(frozen=True)
+class ExtractedBlock:
+    """One canonical-reference/type/text record produced by a recipe."""
+
+    block_id: str
+    block_type: str
+    text: str

epub_blocks/package.py

@@ -0,0 +1,179 @@
+from __future__ import annotations
+
+import fnmatch
+import os
+import posixpath
+from collections.abc import Sequence
+from pathlib import Path
+from urllib.parse import unquote
+from xml.etree.ElementTree import Element
+from zipfile import BadZipFile, ZipFile
+
+from .errors import EpubBlocksError
+from .models import EpubPackage, SpineDocument
+from .safety import DEFAULT_SAFETY_LIMITS, EpubArchive, SafetyLimits
+from .xml import parse_xml
+
+StrPath = str | os.PathLike[str]
+
+CONTAINER_NS = "urn:oasis:names:tc:opendocument:xmlns:container"
+OPF_NS = "http://www.idpf.org/2007/opf"
+DC_NS = "http://purl.org/dc/elements/1.1/"
+PACKAGE_MEDIA_TYPE = "application/oebps-package+xml"
+XHTML_MEDIA_TYPE = "application/xhtml+xml"
+
+
+def archive_path(package_path: str, href: str) -> str:
+    """Resolve a manifest href safely within the EPUB archive."""
+
+    href_path = unquote(href.split("#", 1)[0])
+    if not href_path or "\\" in href_path:
+        raise EpubBlocksError(f"invalid EPUB manifest path: {href!r}")
+    resolved = posixpath.normpath(
+        posixpath.join(posixpath.dirname(package_path), href_path)
+    )
+    if resolved == ".." or resolved.startswith(("../", "/")):
+        raise EpubBlocksError(f"EPUB manifest path escapes the archive root: {href!r}")
+    return resolved
+
+
+def _single_child(parent: Element[str], path: str, label: str) -> Element[str]:
+    elements = parent.findall(path)
+    if len(elements) != 1:
+        raise EpubBlocksError(f"EPUB package must contain exactly one {label}")
+    return elements[0]
+
+
+def read_epub_package(epub: EpubArchive) -> EpubPackage:
+    """Read identifiers and spine metadata from a bounded EPUB reader."""
+
+    container_path = "META-INF/container.xml"
+    container = parse_xml(epub.read(container_path), container_path, epub.limits)
+    if container.tag != f"{{{CONTAINER_NS}}}container":
+        raise EpubBlocksError("EPUB container has an unexpected root element")
+    rootfiles = container.findall(
+        f"./{{{CONTAINER_NS}}}rootfiles/{{{CONTAINER_NS}}}rootfile"
+    )
+    package_rootfiles = [
+        rootfile
+        for rootfile in rootfiles
+        if rootfile.get("media-type") == PACKAGE_MEDIA_TYPE
+    ]
+    if not package_rootfiles:
+        raise EpubBlocksError(
+            "EPUB container has no package rootfile with the required media type"
+        )
+    if len(package_rootfiles) > 1:
+        raise EpubBlocksError("EPUB container identifies multiple package rootfiles")
+    full_path = package_rootfiles[0].get("full-path")
+    if not full_path:
+        raise EpubBlocksError("EPUB package rootfile has no full-path")
+    package_path = archive_path("", full_path)
+
+    package = parse_xml(epub.read(package_path), package_path, epub.limits)
+    if package.tag != f"{{{OPF_NS}}}package":
+        raise EpubBlocksError("EPUB package has an unexpected root element")
+
+    metadata = _single_child(package, f"./{{{OPF_NS}}}metadata", "metadata")
+    manifest_element = _single_child(package, f"./{{{OPF_NS}}}manifest", "manifest")
+    spine_element = _single_child(package, f"./{{{OPF_NS}}}spine", "spine")
+
+    identifiers = frozenset(
+        (element.text or "").strip()
+        for element in metadata.findall(f"./{{{DC_NS}}}identifier")
+        if (element.text or "").strip()
+    )
+
+    manifest: dict[str, Element[str]] = {}
+    for item in manifest_element.findall(f"./{{{OPF_NS}}}item"):
+        item_id = item.get("id")
+        if not item_id:
+            raise EpubBlocksError("EPUB manifest item has no id")
+        if item_id in manifest:
+            raise EpubBlocksError(f"EPUB manifest has duplicate id {item_id!r}")
+        if not item.get("href"):
+            raise EpubBlocksError(f"EPUB manifest item {item_id!r} has no href")
+        if not item.get("media-type"):
+            raise EpubBlocksError(f"EPUB manifest item {item_id!r} has no media-type")
+        manifest[item_id] = item
+
+    spine: list[SpineDocument] = []
+    for position, itemref in enumerate(
+        spine_element.findall(f"./{{{OPF_NS}}}itemref"), 1
+    ):
+        item_id = itemref.get("idref")
+        if not item_id or item_id not in manifest:
+            raise EpubBlocksError(
+                f"EPUB spine refers to missing manifest item {item_id!r}"
+            )
+        linear_value = itemref.get("linear", "yes")
+        if linear_value not in {"yes", "no"}:
+            raise EpubBlocksError(
+                f"EPUB spine item {item_id!r} has invalid linear value {linear_value!r}"
+            )
+        item = manifest[item_id]
+        if item.get("media-type") != XHTML_MEDIA_TYPE:
+            continue
+        href = item.get("href")
+        if href is None:  # validated above; narrows the static type
+            raise AssertionError("validated manifest href is missing")
+        properties = frozenset(
+            f"{item.get('properties', '')} {itemref.get('properties', '')}".split()
+        )
+        spine.append(
+            SpineDocument(
+                position=position,
+                path=archive_path(package_path, href),
+                properties=properties,
+                linear=linear_value == "yes",
+            )
+        )
+
+    if not spine:
+        raise EpubBlocksError("EPUB package has no XHTML spine documents")
+    return EpubPackage(package_path, identifiers, tuple(spine))
+
+
+def inspect_epub(
+    epub_path: StrPath,
+    *,
+    limits: SafetyLimits = DEFAULT_SAFETY_LIMITS,
+) -> EpubPackage:
+    """Open an EPUB and return its validated package information."""
+
+    path = Path(epub_path)
+    try:
+        with ZipFile(path) as archive:
+            return read_epub_package(EpubArchive(archive, limits))
+    except BadZipFile as error:
+        raise EpubBlocksError(f"{path}: not a valid ZIP container") from error
+
+
+def matches(value: str, patterns: Sequence[str]) -> bool:
+    """Match case-insensitive globs without operating-system path rewriting."""
+
+    folded = value.casefold()
+    return any(fnmatch.fnmatchcase(folded, pattern.casefold()) for pattern in patterns)
+
+
+def select_spine_documents(
+    package: EpubPackage,
+    include: Sequence[str] | None = None,
+    exclude: Sequence[str] | None = None,
+    *,
+    include_non_linear: bool = False,
+) -> list[SpineDocument]:
+    """Select spine documents, excluding auxiliary non-linear items by default."""
+
+    include_patterns = include or ("*",)
+    exclude_patterns = exclude or ()
+    selected = [
+        document
+        for document in package.spine
+        if (document.linear or include_non_linear)
+        and matches(document.path, include_patterns)
+        and not matches(document.path, exclude_patterns)
+    ]
+    if not selected:
+        raise EpubBlocksError("No EPUB spine documents matched the selection")
+    return selected