pdf-html 0.1.0__py3-none-any.whl

pdf_html/__init__.py

@@ -0,0 +1,7 @@
+"""pdf-html: convert any text-based PDF into beautiful, self-contained HTML.
+
+Structure is inferred deterministically from font metadata and geometry —
+no NLP, no LLM in the core. Text is always verbatim; images are never emitted.
+"""
+
+__version__ = "0.1.0"

pdf_html/ast.py

@@ -0,0 +1,124 @@
+"""Document model (AST).
+
+Document -> Page -> blocks (Heading, Paragraph, ListBlock, TableBlock,
+Callout). Runs carry inline style (bold, italic, color, size, mono,
+superscript) so the renderer can emit semantic tags; text is always verbatim.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from typing import Union
+
+# Axis-aligned bounding box in PDF user-space points: (x0, y0, x1, y1).
+BBox = tuple[float, float, float, float]
+
+
+@dataclass(frozen=True)
+class SpanStyle:
+    """Inline style metadata for a span or run, from PDF font metadata."""
+
+    font: str = ""
+    size: float = 0.0
+    bold: bool = False
+    italic: bool = False
+    mono: bool = False
+    serif: bool = False
+    superscript: bool = False
+    color: int = 0  # sRGB int, 0xRRGGBB
+
+    @property
+    def css_color(self) -> str:
+        """Return the color as a #rrggbb CSS string."""
+        return f"#{self.color & 0xFFFFFF:06x}"
+
+
+@dataclass(frozen=True)
+class Span:
+    """A contiguous run of same-styled text as extracted from the PDF."""
+
+    text: str
+    bbox: BBox
+    style: SpanStyle
+    # Writing direction of the containing line (usually (1, 0)).
+    direction: tuple[float, float] = (1.0, 0.0)
+
+
+# A Run is a Span that survived page analysis into the document tree.
+Run = Span
+
+
+@dataclass
+class Heading:
+    level: int  # 1..6
+    runs: list[Run] = field(default_factory=list)
+
+
+@dataclass
+class Paragraph:
+    runs: list[Run] = field(default_factory=list)
+    align: str = "left"  # left | center | right | justify
+
+
+@dataclass
+class ListItem:
+    runs: list[Run] = field(default_factory=list)
+    items: list["ListItem"] = field(default_factory=list)  # nested children
+
+
+@dataclass
+class ListBlock:
+    ordered: bool = False
+    items: list[ListItem] = field(default_factory=list)
+
+
+@dataclass
+class TableCell:
+    """One table cell; holds fully classified blocks (paragraphs, lists, ...)."""
+
+    blocks: list["Block"] = field(default_factory=list)
+
+
+@dataclass
+class TableBlock:
+    rows: list[list[TableCell]] = field(default_factory=list)
+    header_rows: int = 0
+
+
+@dataclass
+class Callout:
+    runs: list[Run] = field(default_factory=list)
+    kind: str = "note"
+
+
+Block = Union[Heading, Paragraph, ListBlock, TableBlock, Callout]
+
+
+@dataclass
+class Page:
+    number: int  # 1-based
+    width: float
+    height: float
+    blocks: list[Block] = field(default_factory=list)
+
+
+@dataclass
+class DocumentMeta:
+    title: str = ""
+    body_size: float = 0.0
+    body_font: str = ""
+    heading_sizes: list[float] = field(default_factory=list)  # h1..h6 tiers
+    palette: list[int] = field(default_factory=list)  # sRGB ints
+
+
+@dataclass
+class Document:
+    meta: DocumentMeta = field(default_factory=DocumentMeta)
+    pages: list[Page] = field(default_factory=list)
+    scanned_pages: list[int] = field(default_factory=list)  # 1-based numbers
+
+    @property
+    def is_scanned(self) -> bool:
+        """True when every page lacks extractable text."""
+        return bool(self.pages) and len(self.scanned_pages) == len(self.pages)
+

pdf_html/callout_detector.py

@@ -0,0 +1,6 @@
+"""Callout (highlighted aside) detection.
+
+Generic triggers, in priority order: bold/colored lead-in ending in ":",
+boxed/background regions, indentation + smaller size relative to body.
+Planned — not yet implemented.
+"""

pdf_html/cli.py

@@ -0,0 +1,147 @@
+"""Command-line interface.
+
+pdf-html INPUT.pdf -o out.html [--extractor pymupdf|pdftotext]
+    [--style auto|default] [--paginate] [--no-tables] [--no-callouts]
+    [--keep-headers] [--allow-scanned]
+"""
+
+from __future__ import annotations
+
+import argparse
+import sys
+from collections.abc import Sequence
+
+from .ast import Document, DocumentMeta, Page
+from .extractor import PdfToTextExtractor, PyMuPDFExtractor, TextExtractor
+from .header_footer import strip_headers_footers
+from .renderer import render_html
+from .style_profiler import profile_styles
+from .table_reconstructor import merge_continuation_tables, reconstruct_page
+
+EXIT_OK = 0
+EXIT_SCANNED = 2
+
+
+def build_parser() -> argparse.ArgumentParser:
+    parser = argparse.ArgumentParser(
+        prog="pdf-html",
+        description=(
+            "Convert a text-based PDF into a single self-contained HTML file. "
+            "Text is verbatim; images are dropped; style is inferred from "
+            "font metadata and geometry."
+        ),
+    )
+    parser.add_argument("input", help="Path to the input PDF.")
+    parser.add_argument(
+        "-o", "--output", required=True, help="Path of the HTML file to write."
+    )
+    parser.add_argument(
+        "--extractor",
+        choices=["pymupdf", "pdftotext"],
+        default="pymupdf",
+        help="Text extractor backend (default: pymupdf).",
+    )
+    parser.add_argument(
+        "--style",
+        choices=["auto", "default"],
+        default="auto",
+        help=(
+            "'auto' derives CSS from the document's fonts/sizes/colors; "
+            "'default' uses a clean built-in theme (default: auto)."
+        ),
+    )
+    parser.add_argument(
+        "--paginate",
+        action="store_true",
+        help="Wrap each PDF page in a <section class='sheet'> (off by default).",
+    )
+    parser.add_argument(
+        "--no-tables",
+        action="store_true",
+        help=(
+            "Disable table reconstruction; table text flows as regular "
+            "paragraphs in reading order (tables are on by default)."
+        ),
+    )
+    parser.add_argument(
+        "--no-callouts",
+        action="store_true",
+        help="Disable callout detection (accepted; callouts not yet implemented).",
+    )
+    parser.add_argument(
+        "--keep-headers",
+        action="store_true",
+        help="Keep repeated running headers/footers instead of stripping them.",
+    )
+    parser.add_argument(
+        "--allow-scanned",
+        action="store_true",
+        help="Convert scanned/image PDFs instead of exiting non-zero.",
+    )
+    return parser
+
+
+def _extractor(name: str) -> TextExtractor:
+    if name == "pdftotext":
+        return PdfToTextExtractor()  # TODO: planned, raises NotImplementedError
+    return PyMuPDFExtractor()
+
+
+def convert(args: argparse.Namespace) -> Document:
+    """Run the full pipeline for parsed CLI args and return the AST."""
+    result = _extractor(args.extractor).extract(args.input)
+
+    if result.scanned_pages and not args.allow_scanned:
+        print(
+            f"error: {len(result.scanned_pages)} of {len(result.pages)} page(s) "
+            "contain no extractable text — this looks like a scanned PDF.\n"
+            "hint: pre-run OCR (e.g. `ocrmypdf in.pdf out.pdf`) or pass "
+            "--allow-scanned to convert anyway.",
+            file=sys.stderr,
+        )
+        raise SystemExit(EXIT_SCANNED)
+
+    pages = result.pages if args.keep_headers else strip_headers_footers(result.pages)
+    profile = profile_styles(pages)
+    if args.no_tables:
+        for page in pages:
+            page.tables = []  # reconstruct_page falls back to flow classification
+
+    # TODO: callout detection (--no-callouts) once callout_detector lands.
+    doc = Document(
+        meta=DocumentMeta(
+            title=result.title,
+            body_size=profile.body_size,
+            body_font=profile.body_font,
+            heading_sizes=profile.heading_sizes,
+            palette=profile.palette,
+        ),
+        scanned_pages=list(result.scanned_pages),
+    )
+    pages_blocks = [reconstruct_page(raw, profile) for raw in pages]
+    merge_continuation_tables(pages_blocks)
+    for raw, blocks in zip(pages, pages_blocks, strict=True):
+        doc.pages.append(
+            Page(
+                number=raw.number,
+                width=raw.width,
+                height=raw.height,
+                blocks=blocks,
+            )
+        )
+    return doc
+
+
+def main(argv: Sequence[str] | None = None) -> int:
+    args = build_parser().parse_args(argv)
+    doc = convert(args)
+    html_text = render_html(doc, style=args.style, paginate=args.paginate)
+    with open(args.output, "w", encoding="utf-8") as fh:
+        fh.write(html_text)
+    print(f"wrote {args.output} ({len(doc.pages)} page(s))")
+    return EXIT_OK
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())
+

pdf_html/extractor.py

@@ -0,0 +1,272 @@
+"""Layout-aware text extraction.
+
+Defines the TextExtractor ABC (extension point) and the default
+PyMuPDFExtractor, which uses page.get_text("dict") to capture per-span
+size/flags/color/bbox. A dependency-free pdftotext fallback is planned.
+"""
+
+from __future__ import annotations
+
+import re
+from abc import ABC, abstractmethod
+from dataclasses import dataclass, field
+from typing import TYPE_CHECKING
+
+import pymupdf
+
+from .ast import BBox, Page, Span, SpanStyle
+
+if TYPE_CHECKING:
+    from .ast import Document
+
+# Spans whose text is a run of single letters separated by whitespace are
+# usually decorative glyphs (e.g. titles with extra letter spacing). Collapse
+# those artificial spaces back into words when at least this many consecutive
+# single-letter tokens appear.
+SPACED_GLYPH_MIN_TOKENS = 5
+
+# When a span looks like decorative spaced glyphs, a gap between surrounding
+# non-whitespace characters larger than this fraction of the font size is
+# treated as a word boundary; smaller gaps have their space collapsed.
+SPACED_GLYPH_WORD_GAP_THRESHOLD = 0.35
+
+# PyMuPDF span flag bits (see pymupdf docs for TEXT_FONT_* constants).
+FLAG_SUPERSCRIPT = 1
+FLAG_ITALIC = 2
+FLAG_SERIF = 4
+FLAG_MONO = 8
+FLAG_BOLD = 16
+
+
+@dataclass
+class TableRegion:
+    """A table detected on a page: outer bbox plus per-cell bboxes by row."""
+
+    bbox: BBox
+    rows: list[list[BBox | None]] = field(default_factory=list)  # None = merged cell
+    # Detector hint: 1 when the first row of `rows` is a header row.
+    header_rows: int = 0
+
+
+@dataclass
+class RawPage:
+    """Spans extracted from one page, before reading-order analysis."""
+
+    number: int  # 1-based
+    width: float
+    height: float
+    spans: list[Span] = field(default_factory=list)
+    tables: list[TableRegion] = field(default_factory=list)
+
+
+@dataclass
+class ExtractionResult:
+    """Result of extracting an entire PDF."""
+
+    title: str
+    pages: list[RawPage] = field(default_factory=list)
+    scanned_pages: list[int] = field(default_factory=list)  # 1-based
+
+    @property
+    def is_scanned(self) -> bool:
+        """True when every page lacks extractable text spans."""
+        return bool(self.pages) and len(self.scanned_pages) == len(self.pages)
+
+
+class TextExtractor(ABC):
+    """Extension point: turn a PDF into per-page spans with style metadata."""
+
+    @abstractmethod
+    def extract(self, pdf_path: str) -> ExtractionResult:
+        """Extract spans from *pdf_path*; images are never included."""
+
+
+def style_from_flags(
+    flags: int, *, font: str, size: float, color: int
+) -> SpanStyle:
+    """Build a SpanStyle from a PyMuPDF span flags bitmask."""
+    return SpanStyle(
+        font=font,
+        size=size,
+        bold=bool(flags & FLAG_BOLD),
+        italic=bool(flags & FLAG_ITALIC),
+        mono=bool(flags & FLAG_MONO),
+        serif=bool(flags & FLAG_SERIF),
+        superscript=bool(flags & FLAG_SUPERSCRIPT),
+        color=color,
+    )
+
+
+def _overlaps_any(bbox: BBox, rects: list[pymupdf.Rect]) -> bool:
+    """True when *bbox* intersects any rect in *rects*."""
+    rect = pymupdf.Rect(bbox)
+    return any(rect.intersects(r) for r in rects)
+
+
+def _looks_spaced_glyphs(text: str) -> bool:
+    """True when *text* looks like decorative single-glyph spacing."""
+    min_repeats = SPACED_GLYPH_MIN_TOKENS - 1
+    pattern = re.compile(r"\w(?:\s+\w){" + str(min_repeats) + r",}")
+    return bool(pattern.search(text))
+
+
+def _span_text_from_chars(chars: list[dict], size: float) -> str:
+    """Build span text from raw character dicts.
+
+    Decorative headings sometimes store each glyph with explicit spaces. When
+    a span matches the spaced-glyph pattern, collapse artificial spaces whose
+    surrounding non-whitespace characters are close together, and preserve
+    larger gaps as word boundaries.
+    """
+    if not chars:
+        return ""
+
+    raw_text = "".join(c["c"] for c in chars)
+    if not _looks_spaced_glyphs(raw_text):
+        return raw_text
+
+    threshold = size * SPACED_GLYPH_WORD_GAP_THRESHOLD
+    parts: list[str] = []
+    for i, char_info in enumerate(chars):
+        char = char_info["c"]
+        if char.strip() == "":
+            prev_idx = next(
+                (j for j in range(i - 1, -1, -1) if chars[j]["c"].strip()), None
+            )
+            next_idx = next(
+                (j for j in range(i + 1, len(chars)) if chars[j]["c"].strip()), None
+            )
+            if prev_idx is None or next_idx is None:
+                parts.append(char)
+                continue
+            prev_char = chars[prev_idx]["c"]
+            next_char = chars[next_idx]["c"]
+            if not (prev_char.isalnum() and next_char.isalnum()):
+                parts.append(" ")
+                continue
+            gap = chars[next_idx]["bbox"][0] - chars[prev_idx]["bbox"][2]
+            if gap > threshold:
+                parts.append(" ")
+            # else: collapse the artificial space
+        else:
+            parts.append(char)
+    return "".join(parts)
+
+
+class PyMuPDFExtractor(TextExtractor):
+    """Default extractor: page.get_text("rawdict") -> spans with style metadata."""
+
+    def extract(self, pdf_path: str) -> ExtractionResult:
+        result = ExtractionResult(title="")
+        with pymupdf.open(pdf_path) as doc:
+            assert doc.metadata is not None
+            result.title = doc.metadata.get("title", "") or ""
+            for page in doc:
+                image_rects = self._image_rects(page)
+                spans = self._page_spans(page, image_rects)
+                number = page.number + 1 # type: ignore
+                if not spans:
+                    result.scanned_pages.append(number)
+                result.pages.append(
+                    RawPage(
+                        number=number,
+                        width=page.rect.width,
+                        height=page.rect.height,
+                        spans=spans,
+                        tables=self._table_regions(page) if spans else [],
+                    )
+                )
+        return result
+
+    @staticmethod
+    def _table_regions(page: pymupdf.Page) -> list[TableRegion]:
+        """Detect tables with Page.find_tables() and record their geometry."""
+        regions: list[TableRegion] = []
+        for table in page.find_tables().tables:
+            rows: list[list[BBox | None]] = [
+                [tuple(cell) if cell is not None else None for cell in row.cells]
+                for row in table.rows
+            ]
+            rows = [row for row in rows if any(c is not None for c in row)]
+            if not rows:
+                continue
+            regions.append(
+                TableRegion(
+                    bbox=tuple(table.bbox),
+                    rows=rows,
+                    header_rows=0 if table.header.external else 1,
+                )
+            )
+        return regions
+
+    @staticmethod
+    def _image_rects(page: pymupdf.Page) -> list[pymupdf.Rect]:
+        rects: list[pymupdf.Rect] = []
+        for img in page.get_images(full=True):
+            rects.extend(page.get_image_rects(img[0]))
+        return rects
+
+    @staticmethod
+    def _page_spans(
+        page: pymupdf.Page, image_rects: list[pymupdf.Rect]
+    ) -> list[Span]:
+        spans: list[Span] = []
+        text_page: dict = page.get_text("rawdict")  # type: ignore[assignment]
+        for block in text_page.get("blocks", []):
+            if block.get("type") != 0:  # 0 = text; 1 = image (dropped)
+                continue
+            for line in block.get("lines", []):
+                direction = tuple(line.get("dir", (1.0, 0.0)))
+                for raw in line.get("spans", []):
+                    text = _span_text_from_chars(
+                        raw.get("chars", []), raw.get("size", 0.0)
+                    )
+                    if not text.strip():
+                        continue
+                    bbox: BBox = tuple(raw["bbox"])  # type: ignore[assignment]
+                    if image_rects and _overlaps_any(bbox, image_rects):
+                        continue  # text painted over an image is dropped too
+                    spans.append(
+                        Span(
+                            text=text,
+                            bbox=bbox,
+                            style=style_from_flags(
+                                raw.get("flags", 0),
+                                font=raw.get("font", ""),
+                                size=raw.get("size", 0.0),
+                                color=raw.get("color", 0),
+                            ),
+                            direction=direction,  # type: ignore[arg-type]
+                        )
+                    )
+        return spans
+
+
+class PdfToTextExtractor(TextExtractor):
+    """Dependency-free pdftotext fallback (reduced fidelity).
+
+    Planned — not yet implemented.
+    """
+
+    def extract(self, pdf_path: str) -> ExtractionResult:  # noqa: ARG002
+        raise NotImplementedError(
+            "PdfToTextExtractor is planned but not implemented yet; "
+            "use --extractor pymupdf."
+        )
+
+
+def extract_to_document(result: ExtractionResult) -> Document:
+    """Adapt an ExtractionResult into the AST Document shell.
+
+    Pages carry no blocks yet; structure detection fills those in later.
+    """
+    from .ast import Document, DocumentMeta  # noqa: PLC0415
+
+    doc = Document(meta=DocumentMeta(title=result.title))
+    doc.scanned_pages = list(result.scanned_pages)
+    for raw in result.pages:
+        doc.pages.append(
+            Page(number=raw.number, width=raw.width, height=raw.height)
+        )
+    return doc
+

pdf_html/header_footer.py

@@ -0,0 +1,118 @@
+"""Repeated header/footer stripping (geometry-based).
+
+Removes spans whose bounding box top edge falls within the top 10% of page
+height, or whose bottom edge falls within the bottom 10% of page height, when
+that span text (digits normalized to #) appears on >= 60% of all pages.
+
+Also detects right-hand sidebars that contain legal boilerplate
+(e.g. copyright notices) and removes them so they do not interleave with
+main-column title/body text.
+"""
+
+from __future__ import annotations
+
+import re
+from collections.abc import Sequence
+
+from .extractor import RawPage
+
+# Vertical band at each page edge, as a fraction of page height, in which
+# spans are header/footer candidates.
+HEADER_FOOTER_PAGE_FRACTION = 0.10
+
+# A candidate is stripped only when its digit-normalized text appears on at
+# least this fraction of all pages.
+MIN_PAGE_OCCURRENCE = 0.60
+
+# Repetition is only meaningful when a document has more than one page;
+# for a single page the ">= 60% of pages" rule would fire on real content
+# (e.g. a report title sitting in the top band).
+MIN_PAGES_FOR_STRIPPING = 2
+
+# Fraction of page width considered the right-hand sidebar. Spans whose left
+# edge sits to the right of (1 - RIGHT_SIDEBAR_FRACTION) * page_width are
+# sidebar candidates.
+RIGHT_SIDEBAR_FRACTION = 0.28
+
+# Legal boilerplate triggers. If a right sidebar contains any of these,
+# the whole sidebar is treated as page furniture.
+_LEGAL_RE = re.compile(
+    r"COPYRIGHT|ALL RIGHTS RESERVED|NO PART OF THIS|PUBLICATION MAY BE|"
+    r"REPRODUCED|TRANSMITTED|PRIOR WRITTEN PERMISSION|INTERNAL REFERENCE CODE",
+    re.IGNORECASE,
+)
+
+_DIGITS = re.compile(r"\d+")
+
+
+def normalize_text(text: str) -> str:
+    """Collapse whitespace and replace each digit run with # for comparison."""
+    return _DIGITS.sub("#", " ".join(text.split()))
+
+
+def _is_candidate(span_bbox: tuple[float, float, float, float], page_height: float) -> bool:
+    """True when the bbox top edge is in the top band or the bottom edge is
+    in the bottom band of the page."""
+    band = page_height * HEADER_FOOTER_PAGE_FRACTION
+    top_edge, bottom_edge = span_bbox[1], span_bbox[3]
+    return top_edge <= band or bottom_edge >= page_height - band
+
+
+def _legal_sidebar_spans(page: RawPage) -> set[int]:
+    """Return the object ids of spans that form a right-hand legal sidebar.
+
+    A legal sidebar is a cluster of spans in the right margin whose combined
+    text matches legal boilerplate patterns. Removing these prevents copyright
+    fragments from interleaving with the main title/body column.
+    """
+    threshold = page.width * (1 - RIGHT_SIDEBAR_FRACTION)
+    sidebar = [s for s in page.spans if s.bbox[0] >= threshold]
+    text = "".join(s.text for s in sidebar)
+    if not _LEGAL_RE.search(text):
+        return set()
+    return {id(s) for s in sidebar}
+
+
+def repeated_boilerplate(pages: Sequence[RawPage]) -> set[str]:
+    """Digit-normalized texts that qualify as repeated headers/footers."""
+    if len(pages) < MIN_PAGES_FOR_STRIPPING:
+        return set()
+    occurrences: dict[str, int] = {}
+    for page in pages:
+        seen_on_page: set[str] = set()
+        for span in page.spans:
+            if _is_candidate(span.bbox, page.height):
+                seen_on_page.add(normalize_text(span.text))
+        for text in seen_on_page:
+            occurrences[text] = occurrences.get(text, 0) + 1
+    threshold = MIN_PAGE_OCCURRENCE * len(pages)
+    return {t for t, count in occurrences.items() if count >= threshold}
+
+
+def strip_headers_footers(pages: Sequence[RawPage]) -> list[RawPage]:
+    """Return new RawPages with repeated headers/footers and legal sidebars removed."""
+    boilerplate = repeated_boilerplate(pages)
+    stripped: list[RawPage] = []
+    for page in pages:
+        legal_sidebar = _legal_sidebar_spans(page)
+        spans = []
+        for s in page.spans:
+            if id(s) in legal_sidebar:
+                continue
+            if (
+                _is_candidate(s.bbox, page.height)
+                and normalize_text(s.text) in boilerplate
+            ):
+                continue
+            spans.append(s)
+        stripped.append(
+            RawPage(
+                number=page.number,
+                width=page.width,
+                height=page.height,
+                spans=spans,
+                tables=page.tables,
+            )
+        )
+    return stripped
+