fibrum-pdf 1.2.0__py3-none-win_amd64.whl

fibrum_pdf/__init__.py

@@ -0,0 +1,53 @@
+"""Fast, structured PDF extraction for Python."""
+
+from __future__ import annotations
+
+import logging
+from importlib import metadata
+from typing import TYPE_CHECKING, Any
+
+if TYPE_CHECKING:
+    from .api import ConversionResult, ExtractionError, to_json, to_markdown
+    from .models import Block, ListItem, Page, Pages, Span, TableCell, TableRow
+
+__all__ = [
+    "Block",
+    "ListItem",
+    "Page",
+    "Pages",
+    "Span",
+    "TableCell",
+    "TableRow",
+    "ExtractionError",
+    "to_json",
+    "to_markdown",
+    "ConversionResult",
+    "__version__",
+]
+logging.getLogger(__name__).addHandler(logging.NullHandler())
+
+
+def __getattr__(name: str) -> Any:
+    if name in {"to_json", "to_markdown"}:
+        from . import api
+
+        return getattr(api, name)
+    if name == "ExtractionError":
+        from .api import ExtractionError
+
+        return ExtractionError
+    if name == "ConversionResult":
+        from .api import ConversionResult
+
+        return ConversionResult
+    if name in {"Block", "ListItem", "Page", "Pages", "Span", "TableCell", "TableRow"}:
+        from . import models
+
+        return getattr(models, name)
+    raise AttributeError(name)
+
+
+try:
+    __version__ = metadata.version("fibrum-pdf")
+except metadata.PackageNotFoundError:
+    __version__ = "0.0.0"

fibrum_pdf/_block_converter.py

@@ -0,0 +1,153 @@
+"""Block to Markdown conversion."""
+
+from __future__ import annotations
+
+import logging
+import re
+from typing import Any, cast
+
+log = logging.getLogger(__name__)
+
+BULLETS = frozenset("•‣⁃⁌⁍∙▪▫●○◦■□▶▸◆◇♦➤\uf0b7\ufffd")
+FMT_MARKERS = ("**", "*", "`", "~~")
+PUNCT = " \n\t.,;:)]/\\-?!"
+STYLES = [
+    ("bold", "**"),
+    ("italic", "*"),
+    ("monospace", "`"),
+    ("strikeout", "~~"),
+    ("subscript", "~"),
+]
+
+
+def _style_span(span: dict[str, Any]) -> str:
+    if not (text := span.get("text", "")):
+        return ""
+    link = (
+        next((span.get(k) for k in ["link", "uri"] if isinstance(span.get(k), str)), "")
+        or ""
+    )
+    if span.get("superscript"):
+        s = text.strip()
+        text = f"[{s}]" if s.isdigit() or re.match(r"^\d+[,\s\d]*$", s) else f"^{text}^"
+        return f"[{text}]({link})" if link else text
+    for key, fmt in STYLES:
+        if span.get(key):
+            text = f"{fmt}{text}{fmt}"
+    return f"[{text}]({link})" if link else text
+
+
+def _join_spans(spans: list[dict[str, Any]]) -> str:
+    parts = []
+    for i, span in enumerate(spans):
+        if not (styled := _style_span(span)):
+            continue
+        if (
+            parts
+            and any(styled.startswith(m) for m in FMT_MARKERS)
+            and parts[-1][-1:] not in " \n\t([/"
+        ):
+            parts.append(" ")
+        parts.append(styled)
+        if (
+            i + 1 < len(spans)
+            and (nxt := spans[i + 1].get("text", ""))
+            and any(styled.endswith(m) for m in FMT_MARKERS)
+            and nxt[0] not in PUNCT
+        ):
+            parts.append(" ")
+    return "".join(parts)
+
+
+def _cell_text(cell: dict[str, Any]) -> str:
+    spans = cast(list[dict[str, Any]], cell.get("spans") or [])
+    text = (
+        " ".join(s.get("text", "") for s in spans).strip()
+        if spans
+        else cell.get("text", "").strip()
+    )
+    return text.replace("|", "\\|").replace("\n", "<br>")
+
+
+def _table(rows: list[dict[str, Any]]) -> str:
+    if (
+        not rows
+        or not (
+            matrix := [[_cell_text(c) for c in row.get("cells", [])] for row in rows]
+        )
+        or not any(matrix[0])
+    ):
+        return ""
+    hdr = matrix[0]
+    lines = [
+        "| " + " | ".join(hdr) + " |",
+        "| " + " | ".join("---" for _ in hdr) + " |",
+    ]
+    for row in matrix[1:]:
+        row = (row + [""] * len(hdr))[: len(hdr)]
+        lines.append("| " + " | ".join(row) + " |")
+    return "\n".join(lines) + "\n"
+
+
+def _list(block: dict[str, Any], text: str) -> str:
+    if items := block.get("items"):
+        lines = []
+        for item in items:
+            if not (body := _join_spans(item.get("spans", [])).strip()):
+                continue
+            prefix = item.get("prefix")
+            marker = (
+                prefix
+                if isinstance(prefix, str) and prefix
+                else "1."
+                if item.get("list_type") == "numbered"
+                else "-"
+            )
+            indent = "    " * max(int(item.get("indent", 0)), 0)
+            lines.append(f"{indent}{marker} {body}")
+        return "\n".join(lines) + "\n" if lines else ""
+    return (
+        "\n".join(f"- {ln.strip()}" for ln in text.split("\n") if ln.strip()) + "\n"
+        if text
+        else ""
+    )
+
+
+def _plain_text(block: dict[str, Any]) -> str:
+    return (block.get("text") or "").strip() or "".join(
+        str(span.get("text", "")) for span in block.get("spans", [])
+    ).strip()
+
+
+def _heading(block: dict[str, Any]) -> str:
+    level = max(1, min(int(block.get("level") or 1), 6))
+    return f"{'#' * level} {_plain_text(block)}\n"
+
+
+def _code(block: dict[str, Any]) -> str:
+    text = _plain_text(block)
+    longest_run = max((len(run) for run in re.findall(r"`+", text)), default=0)
+    fence = "`" * max(3, longest_run + 1)
+    return f"{fence}\n{text}\n{fence}\n"
+
+
+def block_to_markdown(block: dict[str, Any]) -> str:
+    typ = block.get("type", "")
+    text = block.get("text", "").strip() or _join_spans(block.get("spans", []))
+    if not text and typ not in ("table", "list"):
+        return ""
+
+    if typ == "heading":
+        return _heading(block)
+    if typ in ("paragraph", "text"):
+        return f"{text}\n"
+    if typ == "code":
+        return _code(block)
+    if typ == "table":
+        return _table(block.get("rows", []))
+    if typ == "list":
+        return _list(block, text)
+    if typ == "figure":
+        return f"![Figure]({block.get('text', 'figure')})\n"
+    log.debug("skipping block type=%s", typ)
+    return ""

fibrum_pdf/_cffi.py

@@ -0,0 +1,82 @@
+"""ctypes bindings and native library discovery."""
+
+from __future__ import annotations
+
+import ctypes
+import logging
+import os
+import sys
+from functools import lru_cache
+from pathlib import Path
+
+log = logging.getLogger(__name__)
+_dll_directory_handles: list[object] = []
+
+
+def _library_name() -> str:
+    return {
+        "darwin": "libtomd.dylib",
+        "win32": "libtomd.dll",
+    }.get(sys.platform, "libtomd.so")
+
+
+@lru_cache(maxsize=1)
+def find_library() -> Path | None:
+    """Find the bundled, locally built, or explicitly configured bridge."""
+    if configured := os.environ.get("FIBRUMPDF_LIB"):
+        path = Path(configured).expanduser()
+        if path.is_file():
+            return path.resolve()
+
+    package = Path(__file__).resolve().parent
+    project, name = package.parent, _library_name()
+    candidates = [package / "lib" / name, project / "lib" / name]
+    candidates.extend((project / "build").glob(f"lib*/fibrum_pdf/lib/{name}"))
+    for path in candidates:
+        if path.is_file():
+            log.debug("found native library: %s", path)
+            return path.resolve()
+    return None
+
+
+def _prepare_dependencies(path: Path) -> None:
+    directories = (
+        path.parent,
+        Path(__file__).resolve().parent.parent / "lib" / "mupdf",
+    )
+    if sys.platform == "win32":
+        for directory in directories:
+            if directory.is_dir() and hasattr(os, "add_dll_directory"):
+                _dll_directory_handles.append(os.add_dll_directory(str(directory)))
+        return
+
+    pattern = "libmupdf*.dylib*" if sys.platform == "darwin" else "libmupdf.so*"
+    for directory in directories:
+        for dependency in sorted(directory.glob(pattern), reverse=True):
+            try:
+                ctypes.CDLL(str(dependency), mode=ctypes.RTLD_GLOBAL)
+                return
+            except OSError:
+                continue
+
+
+def _configure_abi(library: ctypes.CDLL) -> None:
+    library.pdf_to_json.argtypes = [ctypes.c_char_p, ctypes.c_char_p]
+    library.pdf_to_json.restype = ctypes.c_int
+    if hasattr(library, "pdf_to_json_with_error"):
+        library.pdf_to_json_with_error.argtypes = [ctypes.c_char_p, ctypes.c_char_p]
+        library.pdf_to_json_with_error.restype = ctypes.c_void_p
+        library.free_string.argtypes = [ctypes.c_void_p]
+        library.free_string.restype = None
+
+
+@lru_cache(maxsize=1)
+def load_library(path: Path) -> ctypes.CDLL:
+    """Load and configure the native bridge at ``path``."""
+    _prepare_dependencies(path)
+    try:
+        library = ctypes.CDLL(str(path))
+    except OSError as error:
+        raise RuntimeError(f"failed to load libtomd: {error}") from error
+    _configure_abi(library)
+    return library

fibrum_pdf/api.py

@@ -0,0 +1,183 @@
+"""public api for pdf to json extraction."""
+
+from __future__ import annotations
+
+import json
+import logging
+import os
+import sys
+import tempfile
+import threading
+from contextlib import contextmanager
+from functools import cached_property, lru_cache
+from pathlib import Path
+from collections.abc import Generator
+from typing import TYPE_CHECKING, Any, Iterator, cast
+
+if TYPE_CHECKING:
+    from .models import Page, Pages
+
+log = logging.getLogger(__name__)
+_legacy_capture_lock = threading.Lock()
+
+
+class ExtractionError(Exception):
+    pass
+
+
+@contextmanager
+def _redirect_legacy_c_output() -> Generator[int]:
+    """Capture output from native libraries predating direct error returns."""
+    with _legacy_capture_lock, tempfile.TemporaryFile() as capture:
+        saved = os.dup(1), os.dup(2)
+        try:
+            os.dup2(capture.fileno(), 1)
+            os.dup2(capture.fileno(), 2)
+            yield capture.fileno()
+        finally:
+            sys.stdout.flush()
+            sys.stderr.flush()
+            os.dup2(saved[0], 1)
+            os.dup2(saved[1], 2)
+            os.close(saved[0])
+            os.close(saved[1])
+
+
+def _extract(lib: Any, pdf: Path, output: Path) -> str | None:
+    pdf_bytes, output_bytes = os.fsencode(pdf), os.fsencode(output)
+    if hasattr(lib, "pdf_to_json_with_error"):
+        import ctypes
+
+        error = lib.pdf_to_json_with_error(pdf_bytes, output_bytes)
+        if not error:
+            return None
+        try:
+            return ctypes.string_at(error).decode(errors="replace")
+        finally:
+            lib.free_string(error)
+
+    with _redirect_legacy_c_output() as capture_fd:
+        rc = lib.pdf_to_json(pdf_bytes, output_bytes)
+        if rc == 0:
+            return None
+        os.lseek(capture_fd, 0, os.SEEK_SET)
+        return os.read(capture_fd, 1 << 20).decode(errors="replace").strip() or None
+
+
+@lru_cache(maxsize=1)
+def _lib(path: Path | None = None):
+    from ._cffi import find_library, load_library
+
+    p = path or find_library()
+    if not p or not p.exists():
+        raise ExtractionError(
+            "libtomd not found; reinstall fibrum-pdf or set FIBRUMPDF_LIB"
+        )
+    log.info("using library: %s", p)
+    return load_library(p)
+
+
+class ConversionResult:
+    """Lazy PDF conversion result backed by the generated JSON file."""
+
+    def __init__(self, path: Path):
+        """Reference an extraction result at ``path``."""
+        self.path = path
+        log.debug("result at %s", path)
+
+    def _load(self) -> list[Any]:
+        with open(self.path, encoding="utf-8") as f:
+            data: object = json.load(f)
+            return cast(list[Any], data) if isinstance(data, list) else []
+
+    @cached_property
+    def markdown(self) -> str:
+        return "\n---\n\n".join(page.markdown for page in self if page.markdown)
+
+    def write_markdown(self, output: str | Path) -> Path:
+        """Stream Markdown to ``output`` and replace it atomically."""
+        target = Path(output).resolve()
+        target.parent.mkdir(parents=True, exist_ok=True)
+        temporary: Path | None = None
+        try:
+            with tempfile.NamedTemporaryFile(
+                mode="w",
+                encoding="utf-8",
+                dir=target.parent,
+                prefix=f".{target.name}.",
+                delete=False,
+            ) as stream:
+                temporary = Path(stream.name)
+                first = True
+                for page in self:
+                    if not (markdown := page.markdown):
+                        continue
+                    if not first:
+                        stream.write("\n---\n\n")
+                    stream.write(markdown)
+                    first = False
+            temporary.replace(target)
+        finally:
+            if temporary is not None:
+                temporary.unlink(missing_ok=True)
+        return target
+
+    def collect(self) -> "Pages":
+        from .models import Page, Pages
+
+        pages = Pages([Page(p["data"]) for p in self._load()])
+        log.info("collected %d pages", len(pages))
+        return pages
+
+    def __iter__(self) -> Iterator["Page"]:
+        """Stream parsed pages without loading the entire document."""
+        import ijson
+        from .models import Page
+
+        with open(self.path, "rb") as f:
+            for i, p in enumerate(ijson.items(f, "item")):
+                log.debug("page %d", i + 1)
+                yield Page(p["data"])
+
+    def __repr__(self) -> str:
+        """Return a concise representation containing the output path."""
+        return f"ConversionResult({self.path})"
+
+
+def to_json(
+    pdf_path: str | Path,
+    output: str | Path | None = None,
+    *,
+    lib_path: Path | None = None,
+) -> ConversionResult:
+    pdf = Path(pdf_path).resolve()
+    if not pdf.exists():
+        raise FileNotFoundError(f"pdf not found: {pdf}")
+    out = Path(output).resolve() if output else pdf.with_suffix(".json")
+    if out.exists() and os.path.samefile(pdf, out):
+        raise ValueError("input and output must be different files")
+    out.parent.mkdir(parents=True, exist_ok=True)
+    log.info("extracting %s -> %s", pdf, out)
+    if error := _extract(_lib(lib_path), pdf, out):
+        raise ExtractionError(f"extraction failed: {error}")
+    log.info("done")
+    return ConversionResult(out)
+
+
+def to_markdown(
+    pdf_path: str | Path,
+    output: str | Path | None = None,
+    *,
+    lib_path: Path | None = None,
+) -> Path:
+    """Extract ``pdf_path`` to a Markdown file and return its path."""
+    pdf = Path(pdf_path).resolve()
+    target = Path(output).resolve() if output else pdf.with_suffix(".md")
+    if pdf == target:
+        raise ValueError("input and output must be different files")
+    with tempfile.TemporaryDirectory(prefix="fibrum-pdf-") as directory:
+        result = to_json(pdf, Path(directory) / "document.json", lib_path=lib_path)
+        return result.write_markdown(target)
+
+
+__all__ = ["ExtractionError", "to_json", "to_markdown", "ConversionResult"]

fibrum_pdf/main.py

@@ -0,0 +1,70 @@
+"""cli entry point."""
+
+from __future__ import annotations
+
+import logging
+from enum import Enum
+from pathlib import Path
+from typing import Annotated, Optional
+
+import typer
+
+from .api import ExtractionError, to_json, to_markdown
+
+
+class OutputFormat(str, Enum):
+    """Supported CLI output formats."""
+
+    json = "json"
+    markdown = "markdown"
+
+
+def run(
+    pdf_path: Annotated[
+        Path,
+        typer.Argument(
+            exists=True,
+            dir_okay=False,
+            readable=True,
+            resolve_path=True,
+            help="PDF file to extract",
+        ),
+    ],
+    output: Annotated[
+        Optional[Path],
+        typer.Argument(
+            help="Output path (defaults to the input name with a new suffix)"
+        ),
+    ] = None,
+    output_format: Annotated[
+        OutputFormat,
+        typer.Option("--format", "-f", help="Output format"),
+    ] = OutputFormat.json,
+    verbose: Annotated[
+        bool, typer.Option("--verbose", "-v", help="Enable verbose logging")
+    ] = False,
+) -> None:
+    """Extract structured PDF content to JSON or Markdown."""
+    logging.basicConfig(
+        level=logging.DEBUG if verbose else logging.INFO,
+        format="%(levelname)s: %(message)s",
+        force=True,
+    )
+    try:
+        if output_format is OutputFormat.markdown:
+            path = to_markdown(pdf_path, output)
+        else:
+            path = to_json(pdf_path, output).path
+        logging.getLogger(__name__).info("wrote %s", path)
+    except (FileNotFoundError, ValueError, ExtractionError) as e:
+        typer.secho(f"Error: {e}", fg=typer.colors.RED, err=True)
+        raise typer.Exit(1) from e
+
+
+def main() -> None:
+    """CLI entry point."""
+    typer.run(run)
+
+
+if __name__ == "__main__":
+    main()

fibrum_pdf/models.py

@@ -0,0 +1,85 @@
+"""Models for output of FibrumPDF."""
+
+# ruff: noqa: D101, D102, D107
+from __future__ import annotations
+
+import logging
+from functools import cached_property
+from typing import Any
+
+from pydantic import BaseModel, ConfigDict, Field
+
+log = logging.getLogger(__name__)
+
+
+class Span(BaseModel):
+    text: str
+    font_size: float
+    bold: bool = False
+    italic: bool = False
+    monospace: bool = False
+    strikeout: bool = False
+    superscript: bool = False
+    subscript: bool = False
+    link: str | bool = False
+
+
+class TableCell(BaseModel):
+    bbox: list[float]
+    spans: list[Span] = Field(default_factory=lambda: list[Span]())
+
+
+class TableRow(BaseModel):
+    bbox: list[float]
+    cells: list[TableCell] = Field(default_factory=lambda: list[TableCell]())
+
+
+class ListItem(BaseModel):
+    spans: list[Span] = Field(default_factory=lambda: list[Span]())
+    list_type: str
+    indent: int = 0
+    prefix: str | bool = False
+
+
+class Block(BaseModel):
+    model_config = ConfigDict(extra="allow")
+    type: str
+    bbox: list[float]
+    spans: list[Span] = Field(default_factory=lambda: list[Span]())
+    length: int = 0
+    lines: int | None = None
+    level: int | None = None
+    row_count: int | None = None
+    col_count: int | None = None
+    cell_count: int | None = None
+    rows: list[TableRow] | None = None
+    items: list[ListItem] | None = None
+
+    @cached_property
+    def markdown(self) -> str:
+        from ._block_converter import block_to_markdown
+
+        return block_to_markdown(self.model_dump())
+
+
+class Page(list[Block]):
+    def __init__(self, items: list[Block | dict[str, Any]] | dict[str, Any]):
+        super().__init__()
+        if isinstance(items, dict) and "data" in items:
+            items = items["data"]
+        for item in items or []:
+            self.append(Block(**item) if isinstance(item, dict) else item)  # type: ignore
+        log.debug("page: %d blocks", len(self))
+
+    @cached_property
+    def markdown(self) -> str:
+        return "\n".join(b.markdown for b in self if b.markdown)
+
+
+class Pages(list[Page]):
+    def __init__(self, pages: list[Page] | None = None):
+        super().__init__(pages or [])
+
+    @cached_property
+    def markdown(self) -> str:
+        return "\n---\n\n".join(p.markdown for p in self if p.markdown)