rc-docparser 0.2.0__py3-none-any.whl

docparser/epub.py

@@ -0,0 +1,349 @@
+"""EPUB parser.
+
+Walks the EPUB spine in reading order, runs a BeautifulSoup structural walk
+over each chapter (headings / paragraphs / lists / tables / images), and
+extracts embedded images to the asset directory. Embedded images can be
+captioned via a ``captioner`` callable.
+
+Requires the ``[epub]`` extra: ``pip install 'docparser[epub]'`` (which also
+pulls in BeautifulSoup from the ``[html]`` extra).
+"""
+from __future__ import annotations
+
+import re
+from collections.abc import Callable
+from pathlib import Path
+from typing import Any
+
+from .common import (
+    WorkspaceLayout,
+    bytes_sha1,
+    file_sha1,
+    truncate,
+    utc_now_iso,
+    write_json,
+    write_text,
+)
+
+_HEADING_RE = re.compile(r"^h([1-6])$", re.IGNORECASE)
+
+
+def _import_deps():
+    try:
+        import ebooklib  # type: ignore
+        from bs4 import BeautifulSoup  # type: ignore
+        from ebooklib import epub  # type: ignore
+    except ImportError as exc:  # pragma: no cover - optional dep guard
+        raise ImportError(
+            "docparser.epub.parse_epub requires the [epub] extra. "
+            "Install with: pip install 'docparser[epub]'"
+        ) from exc
+    return ebooklib, epub, BeautifulSoup
+
+
+def _ext_for(media_type: str, name: str) -> str:
+    mapping = {
+        "image/png": "png",
+        "image/jpeg": "jpg",
+        "image/gif": "gif",
+        "image/svg+xml": "svg",
+        "image/webp": "webp",
+    }
+    if media_type in mapping:
+        return mapping[media_type]
+    return Path(name or "").suffix.lstrip(".").lower() or "png"
+
+
+def parse_epub(
+    source: Path | str,
+    layout: WorkspaceLayout | None = None,
+    *,
+    captioner: Callable[..., dict[str, Any]] | None = None,
+    write_outputs: bool = True,
+) -> dict[str, Any]:
+    """Parse an EPUB into Markdown + JSON + images."""
+    ebooklib, epub, BeautifulSoup = _import_deps()
+
+    source = Path(source)
+    layout = layout or WorkspaceLayout()
+    real_source = source.resolve()
+    book = epub.read_epub(str(real_source))
+
+    out_dir = layout.parsed_dir_for(source)
+    asset_dir = layout.assets_dir_for(source)
+    if write_outputs:
+        out_dir.mkdir(parents=True, exist_ok=True)
+        asset_dir.mkdir(parents=True, exist_ok=True)
+
+    # ---- extract embedded images up front, keyed by (normalized) href -----
+    image_by_name: dict[str, dict[str, Any]] = {}
+    images: list[dict[str, Any]] = []
+    image_caption_results: dict[str, dict[str, Any]] = {}
+    image_seq = 0
+    for item in book.get_items_of_type(ebooklib.ITEM_IMAGE):
+        blob = item.get_content()
+        if not blob:
+            continue
+        image_seq += 1
+        sha = bytes_sha1(blob)
+        ext = _ext_for(getattr(item, "media_type", "") or "", item.get_name())
+        asset_name = f"img-{image_seq:03d}-{sha[:10]}.{ext}"
+        asset_path = asset_dir / asset_name
+        if write_outputs and not asset_path.exists():
+            asset_path.write_bytes(blob)
+        rel = layout.relpath_from_parsed(asset_path, source)
+        rec = {
+            "seq": image_seq,
+            "sha1": sha,
+            "ext": ext,
+            "name": item.get_name(),
+            "asset_path": rel,
+            "blob": blob,
+            "mime": getattr(item, "media_type", "") or f"image/{ext}",
+        }
+        key = Path(item.get_name()).name.lower()
+        image_by_name[key] = rec
+        images.append(rec)
+
+    def _resolve_image(src: str) -> dict[str, Any] | None:
+        if not src:
+            return None
+        return image_by_name.get(Path(src.split("#")[0]).name.lower())
+
+    # ---- metadata ---------------------------------------------------------
+    def _meta(field: str) -> str:
+        try:
+            vals = book.get_metadata("DC", field)
+            if vals:
+                return str(vals[0][0])
+        except Exception:
+            pass
+        return ""
+
+    title = _meta("title") or source.stem
+    author = _meta("creator")
+    language = _meta("language")
+
+    # ---- structural walk per spine document -------------------------------
+    blocks: list[dict[str, Any]] = []
+    section_stack: list[str] = []
+
+    def push_heading(level: int, text: str) -> None:
+        while len(section_stack) >= level:
+            section_stack.pop()
+        section_stack.append(text)
+        blocks.append(
+            {"kind": "heading", "level": level, "text": text, "section_path": list(section_stack)}
+        )
+
+    spine_ids = [sid for sid, _ in book.spine]
+    items = []
+    for sid in spine_ids:
+        it = book.get_item_with_id(sid)
+        if it is not None and it.get_type() == ebooklib.ITEM_DOCUMENT:
+            items.append(it)
+    if not items:
+        items = list(book.get_items_of_type(ebooklib.ITEM_DOCUMENT))
+
+    for doc_item in items:
+        content = doc_item.get_content()
+        if not content:
+            continue
+        soup = BeautifulSoup(content, "html.parser")
+        for tag in soup(["script", "style"]):
+            tag.decompose()
+        body = soup.body or soup
+        seen_lists: set[int] = set()
+        for el in body.find_all(True, recursive=True):
+            name = (el.name or "").lower()
+            m = _HEADING_RE.match(name)
+            if m:
+                text = el.get_text(" ", strip=True)
+                if text:
+                    push_heading(int(m.group(1)), text)
+                continue
+            if name == "p":
+                text = el.get_text(" ", strip=True)
+                if text:
+                    blocks.append(
+                        {"kind": "paragraph", "text": text, "section_path": list(section_stack)}
+                    )
+                continue
+            if name in {"ul", "ol"}:
+                if id(el) in seen_lists:
+                    continue
+                seen_lists.add(id(el))
+                li_items = [li.get_text(" ", strip=True) for li in el.find_all("li", recursive=False)]
+                li_items = [i for i in li_items if i]
+                if li_items:
+                    blocks.append(
+                        {
+                            "kind": "list",
+                            "ordered": name == "ol",
+                            "items": li_items,
+                            "section_path": list(section_stack),
+                        }
+                    )
+                continue
+            if name == "table":
+                rows: list[list[str]] = []
+                for tr in el.find_all("tr", recursive=True):
+                    cells = [
+                        td.get_text(" ", strip=True)
+                        for td in tr.find_all(["td", "th"], recursive=False)
+                    ]
+                    if cells:
+                        rows.append(cells)
+                if rows:
+                    blocks.append(
+                        {"kind": "table", "rows": rows, "section_path": list(section_stack)}
+                    )
+                continue
+            if name in {"img", "image"}:
+                src = el.get("src") or el.get("href") or el.get("xlink:href") or ""
+                img_rec = _resolve_image(src)
+                blocks.append(
+                    {
+                        "kind": "image",
+                        "src": src,
+                        "alt": el.get("alt") or "",
+                        "image_seq": img_rec["seq"] if img_rec else None,
+                        "asset_path": img_rec["asset_path"] if img_rec else None,
+                        "section_path": list(section_stack),
+                    }
+                )
+                continue
+
+    # ---- caption images that were actually referenced ---------------------
+    referenced_seqs = {b["image_seq"] for b in blocks if b.get("kind") == "image" and b.get("image_seq")}
+    if captioner is not None:
+        ctx_by_seq: dict[int, str] = {}
+        last_text = ""
+        for b in blocks:
+            if b["kind"] in {"paragraph", "heading"}:
+                last_text = b.get("text", "")
+            elif b["kind"] == "image" and b.get("image_seq"):
+                ctx_by_seq.setdefault(b["image_seq"], last_text)
+        for rec in images:
+            if rec["seq"] not in referenced_seqs:
+                continue
+            try:
+                cap = captioner(
+                    image_bytes=rec["blob"],
+                    mime=rec["mime"],
+                    doc_name=source.name,
+                    nearby_caption="",
+                    context=ctx_by_seq.get(rec["seq"], ""),
+                )
+            except Exception as exc:
+                cap = {"error": str(exc)}
+            image_caption_results[str(rec["seq"])] = cap or {}
+
+    # ---- markdown rendering -----------------------------------------------
+    md_lines = [
+        f"# {title}",
+        "",
+        f"> Source: `{source.name}` \u00b7 sha1 `{file_sha1(real_source)[:12]}` "
+        f"\u00b7 parsed `{utc_now_iso()}`",
+    ]
+    meta_parts = []
+    if author:
+        meta_parts.append(f"author: {author}")
+    if language:
+        meta_parts.append(f"language: {language}")
+    if meta_parts:
+        md_lines.append("> " + " \u00b7 ".join(meta_parts))
+    md_lines.append("")
+
+    for b in blocks:
+        kind = b["kind"]
+        if kind == "heading":
+            md_lines.append(f"{'#' * min(6, b['level'] + 1)} {b['text']}")
+            md_lines.append("")
+        elif kind == "paragraph":
+            md_lines.append(b["text"])
+            md_lines.append("")
+        elif kind == "list":
+            for item in b["items"]:
+                md_lines.append(f"- {item}")
+            md_lines.append("")
+        elif kind == "table":
+            rows = b["rows"]
+            ncols = max(len(r) for r in rows)
+            header = rows[0]
+            md_lines.append(
+                "| "
+                + " | ".join(
+                    (header[c] if c < len(header) else "").replace("|", "\\|") for c in range(ncols)
+                )
+                + " |"
+            )
+            md_lines.append("| " + " | ".join(["---"] * ncols) + " |")
+            for row in rows[1:]:
+                md_lines.append(
+                    "| "
+                    + " | ".join(
+                        (row[c] if c < len(row) else "").replace("|", "\\|") for c in range(ncols)
+                    )
+                    + " |"
+                )
+            md_lines.append("")
+        elif kind == "image":
+            rel = b.get("asset_path") or b.get("src", "")
+            cap = image_caption_results.get(str(b.get("image_seq"))) or {}
+            alt = cap.get("caption") or b.get("alt") or "image"
+            md_lines.append(f"![{alt}]({rel})")
+            if cap.get("description"):
+                md_lines.append("")
+                md_lines.append(f"> **VLM caption.** {cap.get('caption','')}")
+                md_lines.append(">")
+                md_lines.append(f"> {cap.get('description','')}")
+                if cap.get("tags"):
+                    md_lines.append(">")
+                    md_lines.append("> *Tags:* " + ", ".join(cap["tags"]))
+            md_lines.append("")
+
+    md_text = "\n".join(md_lines).rstrip() + "\n"
+
+    images_json = []
+    for rec in images:
+        d = {k: v for k, v in rec.items() if k != "blob"}
+        d["semantic"] = image_caption_results.get(str(rec["seq"]))
+        d["referenced"] = rec["seq"] in referenced_seqs
+        images_json.append(d)
+
+    json_payload = {
+        "source": {
+            "filename": source.name,
+            "absolute_path": str(real_source),
+            "sha1": file_sha1(real_source),
+            "size_bytes": real_source.stat().st_size,
+            "kind": "epub",
+        },
+        "parsed_at": utc_now_iso(),
+        "metadata": {"title": title, "author": author, "language": language},
+        "blocks": blocks,
+        "images": images_json,
+        "stats": {
+            "n_blocks": len(blocks),
+            "n_chapters": len(items),
+            "n_headings": sum(1 for b in blocks if b["kind"] == "heading"),
+            "n_paragraphs": sum(1 for b in blocks if b["kind"] == "paragraph"),
+            "n_lists": sum(1 for b in blocks if b["kind"] == "list"),
+            "n_tables": sum(1 for b in blocks if b["kind"] == "table"),
+            "n_images": len(images),
+            "n_captioned_images": sum(
+                1
+                for v in image_caption_results.values()
+                if v and not v.get("error") and v.get("caption")
+            ),
+        },
+    }
+
+    _ = truncate  # symmetry with other parsers
+
+    if write_outputs:
+        write_text(out_dir / "document.md", md_text)
+        write_json(out_dir / "document.json", json_payload)
+
+    return json_payload

docparser/html.py

@@ -0,0 +1,322 @@
+"""HTML parser.
+
+Two-tier strategy:
+
+1. **Article extraction** via `trafilatura` for the main body (drops nav,
+   sidebars, ads).
+2. **Structural fallback** via BeautifulSoup that walks the DOM and emits
+   typed blocks (heading/paragraph/list/table/image) when trafilatura returns
+   nothing useful or when the caller wants the full structure.
+
+Requires the ``[html]`` extra: ``pip install 'docparser[html]'``.
+"""
+from __future__ import annotations
+
+import re
+from collections.abc import Callable
+from pathlib import Path
+from typing import Any
+from urllib.parse import urlparse
+
+from .common import (
+    WorkspaceLayout,
+    file_sha1,
+    truncate,
+    utc_now_iso,
+    write_json,
+    write_text,
+)
+
+_HEADING_RE = re.compile(r"^h([1-6])$", re.IGNORECASE)
+
+
+def _import_deps():
+    try:
+        import trafilatura  # type: ignore
+        from bs4 import BeautifulSoup  # type: ignore
+    except ImportError as exc:  # pragma: no cover
+        raise ImportError(
+            "docparser.html.parse_html requires the [html] extra. "
+            "Install with: pip install 'docparser[html]'"
+        ) from exc
+    return trafilatura, BeautifulSoup
+
+
+def parse_html(
+    source: Path | str,
+    layout: WorkspaceLayout | None = None,
+    *,
+    captioner: Callable[..., dict[str, Any]] | None = None,
+    write_outputs: bool = True,
+    use_trafilatura: bool = True,
+) -> dict[str, Any]:
+    """Parse an HTML file (or http(s) URL) into Markdown + JSON.
+
+    Parameters
+    ----------
+    source : Path | str
+        Local path or full URL. URLs are fetched via ``trafilatura`` (which
+        falls back to ``urllib`` if needed).
+    use_trafilatura : bool
+        If True (default), prefer trafilatura's article extraction. The
+        structural BS4 walk is always run as the fallback / structural source.
+    """
+    trafilatura, BeautifulSoup = _import_deps()
+
+    is_url = isinstance(source, str) and re.match(r"^https?://", source) is not None
+    real_source: Path | None
+    if is_url:
+        url = str(source)
+        downloaded = trafilatura.fetch_url(url)
+        if not downloaded:
+            raise RuntimeError(f"Could not fetch URL: {url}")
+        html_text = downloaded
+        # Use the URL's path for layout slugs; fall back to host
+        host = urlparse(url).netloc.replace(":", "_")
+        path_part = urlparse(url).path.strip("/").replace("/", "_") or "index"
+        synthetic_name = f"{host}_{path_part}.html"
+        layout = layout or WorkspaceLayout()
+        out_dir = layout.parsed_dir / layout.parsed_dir_for(Path(synthetic_name)).name
+        asset_dir = layout.assets_dir_for(Path(synthetic_name))
+        source_filename = synthetic_name
+        absolute_path = url
+        size_bytes = len(html_text.encode("utf-8"))
+        sha = ""  # we hash the fetched bytes below
+        import hashlib
+
+        sha = hashlib.sha1(html_text.encode("utf-8")).hexdigest()
+        real_source = None
+    else:
+        real_source = Path(source).resolve()
+        html_text = real_source.read_text(encoding="utf-8", errors="replace")
+        layout = layout or WorkspaceLayout()
+        out_dir = layout.parsed_dir_for(real_source)
+        asset_dir = layout.assets_dir_for(real_source)
+        source_filename = real_source.name
+        absolute_path = str(real_source)
+        size_bytes = real_source.stat().st_size
+        sha = file_sha1(real_source)
+
+    if write_outputs:
+        out_dir.mkdir(parents=True, exist_ok=True)
+        asset_dir.mkdir(parents=True, exist_ok=True)
+
+    # 1. Article-grade markdown via trafilatura --------------------------------
+    article_md: str | None = None
+    article_meta: dict[str, Any] = {}
+    if use_trafilatura:
+        try:
+            article_md = trafilatura.extract(
+                html_text,
+                output_format="markdown",
+                include_comments=False,
+                include_tables=True,
+                include_links=True,
+                with_metadata=False,
+            )
+            meta = trafilatura.extract_metadata(html_text)
+            if meta is not None:
+                article_meta = {
+                    k: getattr(meta, k, None)
+                    for k in (
+                        "title",
+                        "author",
+                        "date",
+                        "sitename",
+                        "url",
+                        "description",
+                        "categories",
+                        "tags",
+                    )
+                }
+        except Exception:
+            article_md = None
+
+    # 2. Structural walk via BeautifulSoup --------------------------------------
+    soup = BeautifulSoup(html_text, "html.parser")
+    for tag in soup(["script", "style", "noscript"]):
+        tag.decompose()
+
+    blocks: list[dict[str, Any]] = []
+    section_stack: list[str] = []
+
+    def push_heading(level: int, text: str) -> None:
+        nonlocal section_stack
+        while len(section_stack) >= level:
+            section_stack.pop()
+        section_stack.append(text)
+        blocks.append(
+            {
+                "kind": "heading",
+                "level": level,
+                "text": text,
+                "section_path": list(section_stack),
+            }
+        )
+
+    title_tag = soup.find("title")
+    if title_tag and title_tag.get_text(strip=True):
+        push_heading(1, title_tag.get_text(strip=True))
+
+    # walk body in document order
+    body = soup.body or soup
+    seen_lists: set[int] = set()
+    for el in body.find_all(True, recursive=True):
+        name = (el.name or "").lower()
+        m = _HEADING_RE.match(name)
+        if m:
+            level = int(m.group(1))
+            text = el.get_text(" ", strip=True)
+            if text:
+                push_heading(level, text)
+            continue
+        if name == "p":
+            text = el.get_text(" ", strip=True)
+            if text:
+                blocks.append(
+                    {"kind": "paragraph", "text": text, "section_path": list(section_stack)}
+                )
+            continue
+        if name in {"ul", "ol"}:
+            if id(el) in seen_lists:
+                continue
+            seen_lists.add(id(el))
+            items = [li.get_text(" ", strip=True) for li in el.find_all("li", recursive=False)]
+            items = [i for i in items if i]
+            if items:
+                blocks.append(
+                    {
+                        "kind": "list",
+                        "ordered": name == "ol",
+                        "items": items,
+                        "section_path": list(section_stack),
+                    }
+                )
+            continue
+        if name == "table":
+            rows: list[list[str]] = []
+            for tr in el.find_all("tr", recursive=True):
+                cells = [
+                    td.get_text(" ", strip=True)
+                    for td in tr.find_all(["td", "th"], recursive=False)
+                ]
+                if cells:
+                    rows.append(cells)
+            if rows:
+                blocks.append(
+                    {"kind": "table", "rows": rows, "section_path": list(section_stack)}
+                )
+            continue
+        if name == "img":
+            src = el.get("src") or ""
+            alt = el.get("alt") or ""
+            blocks.append(
+                {
+                    "kind": "image",
+                    "src": src,
+                    "alt": alt,
+                    "section_path": list(section_stack),
+                }
+            )
+            continue
+
+    # 3. Markdown rendering ---------------------------------------------------
+    md_lines: list[str] = [
+        f"# {article_meta.get('title') or Path(source_filename).stem}",
+        "",
+        f"> Source: `{source_filename}` \u00b7 sha1 `{sha[:12]}` \u00b7 parsed `{utc_now_iso()}`",
+        "",
+    ]
+    if article_meta:
+        meta_parts = []
+        if article_meta.get("author"):
+            meta_parts.append(f"author: {article_meta['author']}")
+        if article_meta.get("date"):
+            meta_parts.append(f"date: {article_meta['date']}")
+        if article_meta.get("sitename"):
+            meta_parts.append(f"site: {article_meta['sitename']}")
+        if article_meta.get("url"):
+            meta_parts.append(f"url: {article_meta['url']}")
+        if meta_parts:
+            md_lines.append("> " + " \u00b7 ".join(meta_parts))
+            md_lines.append("")
+
+    if article_md and article_md.strip():
+        md_lines.append("## Article (trafilatura)")
+        md_lines.append("")
+        md_lines.append(article_md.strip())
+        md_lines.append("")
+        md_lines.append("---")
+        md_lines.append("")
+
+    md_lines.append("## Structural extraction")
+    md_lines.append("")
+    for b in blocks:
+        kind = b["kind"]
+        if kind == "heading":
+            level = b["level"]
+            md_lines.append(f"{'#' * min(6, level + 2)} {b['text']}")
+            md_lines.append("")
+        elif kind == "paragraph":
+            md_lines.append(b["text"])
+            md_lines.append("")
+        elif kind == "list":
+            for item in b["items"]:
+                md_lines.append(f"- {item}")
+            md_lines.append("")
+        elif kind == "table":
+            rows = b["rows"]
+            ncols = max(len(r) for r in rows)
+            header = rows[0]
+            header_cells = [
+                (header[c] if c < len(header) else "").replace("|", "\\|").replace("\n", " ")
+                for c in range(ncols)
+            ]
+            md_lines.append("| " + " | ".join(header_cells) + " |")
+            md_lines.append("| " + " | ".join(["---"] * ncols) + " |")
+            for row in rows[1:]:
+                cells = [
+                    (row[c] if c < len(row) else "").replace("|", "\\|").replace("\n", " ")
+                    for c in range(ncols)
+                ]
+                md_lines.append("| " + " | ".join(cells) + " |")
+            md_lines.append("")
+        elif kind == "image":
+            md_lines.append(f"![{b.get('alt') or 'image'}]({b.get('src','')})")
+            md_lines.append("")
+
+    md_text = "\n".join(md_lines).rstrip() + "\n"
+
+    json_payload = {
+        "source": {
+            "filename": source_filename,
+            "absolute_path": absolute_path,
+            "sha1": sha,
+            "size_bytes": size_bytes,
+            "kind": "html",
+        },
+        "parsed_at": utc_now_iso(),
+        "metadata": article_meta,
+        "article_markdown": article_md,
+        "blocks": blocks,
+        "stats": {
+            "n_blocks": len(blocks),
+            "n_headings": sum(1 for b in blocks if b["kind"] == "heading"),
+            "n_paragraphs": sum(1 for b in blocks if b["kind"] == "paragraph"),
+            "n_lists": sum(1 for b in blocks if b["kind"] == "list"),
+            "n_tables": sum(1 for b in blocks if b["kind"] == "table"),
+            "n_images": sum(1 for b in blocks if b["kind"] == "image"),
+            "article_chars": len(article_md or ""),
+        },
+    }
+
+    # captioner is supported here only for completeness; HTML images are usually
+    # remote URLs we don't fetch by default.
+    _ = captioner
+    _ = truncate  # silence unused in some configurations
+
+    if write_outputs:
+        write_text(out_dir / "document.md", md_text)
+        write_json(out_dir / "document.json", json_payload)
+
+    return json_payload