pdf2mj 0.1.0__py3-none-any.whl

pdf2mj/__init__.py

@@ -0,0 +1,3 @@
+"""PDF2MJ — convert PDF documents to Markdown and structured JSON."""
+
+__version__ = "0.1.0"

pdf2mj/chunker.py

@@ -0,0 +1,58 @@
+"""Text chunking for RAG pipelines."""
+
+from __future__ import annotations
+
+import re
+
+from pdf2mj.models import Chunk, Document
+
+
+def _estimate_tokens(text: str) -> int:
+    return len(re.findall(r"\S+", text))
+
+
+def chunk_document(
+    document: Document,
+    chunk_size: int = 1000,
+    overlap: int = 200,
+) -> list[Chunk]:
+    """Split document text into overlapping chunks with page references."""
+    chunks: list[Chunk] = []
+    chunk_index = 0
+
+    for page in document.pages:
+        page_text_parts: list[str] = []
+        for block in page.blocks:
+            if block.type in ("image",):
+                continue
+            text = block.text.strip()
+            if text:
+                page_text_parts.append(text)
+
+        full_text = "\n\n".join(page_text_parts)
+        if not full_text.strip():
+            continue
+
+        words = full_text.split()
+        if not words:
+            continue
+
+        start = 0
+        while start < len(words):
+            end = min(start + chunk_size, len(words))
+            chunk_words = words[start:end]
+            chunk_text = " ".join(chunk_words)
+            chunk_index += 1
+            chunks.append(
+                Chunk(
+                    chunk_id=f"c{chunk_index}",
+                    page=page.page_number,
+                    text=chunk_text,
+                    tokens=_estimate_tokens(chunk_text),
+                )
+            )
+            if end >= len(words):
+                break
+            start = max(0, end - overlap)
+
+    return chunks

pdf2mj/cli.py

@@ -0,0 +1,207 @@
+"""PDF2MJ command-line interface."""
+
+from __future__ import annotations
+
+import sys
+from pathlib import Path
+from typing import Annotated, Optional
+
+import typer
+from rich.console import Console
+from rich.panel import Panel
+
+from pdf2mj.chunker import chunk_document
+from pdf2mj.console_util import make_console
+from pdf2mj.converter import PDFConversionError, parse_pdf
+from pdf2mj.doctor import run_doctor
+from pdf2mj.json_export import write_chunks, write_json, write_metadata_json
+from pdf2mj.markdown import write_markdown
+from pdf2mj.ocr import ocr_available
+from pdf2mj.welcome import show_welcome, show_welcome_if_needed
+
+console = make_console()
+
+app = typer.Typer(
+    name="pdf2mj",
+    help="Convert PDF documents to Markdown and structured JSON.",
+    no_args_is_help=False,
+    invoke_without_command=True,
+    add_completion=False,
+    rich_markup_mode="rich",
+)
+
+convert_app = typer.Typer(
+    help="Convert a PDF to Markdown and/or structured JSON.",
+    add_completion=False,
+)
+
+_SUBCOMMANDS = frozenset({"welcome", "doctor"})
+
+
+@app.callback()
+def root(ctx: typer.Context) -> None:
+    """Show first-run welcome and help when invoked with no subcommand."""
+    if ctx.invoked_subcommand is not None:
+        return
+    show_welcome_if_needed(console=console)
+    typer.echo(ctx.get_help())
+
+
+@app.command()
+def welcome() -> None:
+    """Display the onboarding welcome screen."""
+    show_welcome(console=console, mark_seen=True)
+
+
+@app.command()
+def doctor(
+    output_dir: Annotated[
+        Optional[Path],
+        typer.Option(
+            "--output",
+            "-o",
+            help="Directory to verify write permissions for.",
+            file_okay=False,
+        ),
+    ] = None,
+) -> None:
+    """Verify environment, dependencies, and optional OCR support."""
+    code = run_doctor(console=console, output_dir=output_dir)
+    raise typer.Exit(code=code)
+
+
+@convert_app.command()
+def convert(
+    pdf: Annotated[Path, typer.Argument(help="Path to the input PDF file.", exists=True)],
+    markdown: Annotated[bool, typer.Option("--markdown/--no-markdown", help="Generate markdown.")] = True,
+    json_out: Annotated[bool, typer.Option("--json/--no-json", help="Generate structured JSON.")] = True,
+    ocr: Annotated[bool, typer.Option("--ocr", help="OCR scanned PDF pages.")] = False,
+    extract_images: Annotated[
+        bool, typer.Option("--extract-images", help="Extract embedded images.")
+    ] = False,
+    chunk_size: Annotated[
+        Optional[int], typer.Option("--chunk-size", help="Chunk size (words) for RAG.")
+    ] = None,
+    chunk_overlap: Annotated[
+        int, typer.Option("--chunk-overlap", help="Overlap (words) between chunks.")
+    ] = 200,
+    output: Annotated[
+        Optional[Path], typer.Option("--output", "-o", help="Output directory.")
+    ] = None,
+    verbose: Annotated[bool, typer.Option("--verbose", "-v", help="Detailed logs.")] = False,
+    metadata: Annotated[bool, typer.Option("--metadata", help="Export metadata.json.")] = False,
+    tables: Annotated[bool, typer.Option("--tables/--no-tables", help="Extract tables.")] = True,
+    figures: Annotated[
+        bool, typer.Option("--figures", help="Extract figures (same as --extract-images).")
+    ] = False,
+    all_outputs: Annotated[
+        bool, typer.Option("--all", help="Enable all outputs.")
+    ] = False,
+) -> None:
+    """Convert a PDF to Markdown and/or structured JSON."""
+    if all_outputs:
+        markdown = True
+        json_out = True
+        extract_images = True
+        metadata = True
+        tables = True
+        if chunk_size is None:
+            chunk_size = 1000
+
+    if figures:
+        extract_images = True
+
+    if ocr and not ocr_available():
+        console.print(
+            "[yellow]Warning:[/yellow] OCR requested but pytesseract is not installed. "
+            "Install with: pip install pdf2mj[ocr]"
+        )
+        ocr = False
+
+    stem = pdf.stem
+    out_dir = output if output is not None else pdf.parent
+    out_dir.mkdir(parents=True, exist_ok=True)
+    images_dir = out_dir / "images"
+
+    def log(msg: str, style: str = "green") -> None:
+        console.print(f"[{style}][OK][/] {msg}")
+
+    try:
+        if verbose:
+            console.print(Panel.fit(f"Processing [bold]{pdf.name}[/bold]", title="PDF2MJ"))
+
+        document = parse_pdf(
+            pdf,
+            use_ocr=ocr,
+            extract_images=extract_images,
+            extract_tables=tables,
+            images_output_dir=images_dir if extract_images else None,
+        )
+        log("PDF Loaded")
+
+        if metadata:
+            write_metadata_json(document, out_dir / f"{stem}_metadata.json")
+            log("Metadata Extracted")
+
+        if markdown:
+            md_path = out_dir / f"{stem}.md"
+            write_markdown(document, md_path, pdf_path=pdf, use_pymupdf4llm=True)
+            log("Markdown Generated")
+
+        if json_out:
+            json_path = out_dir / f"{stem}.json"
+            write_json(document, json_path)
+            log("JSON Generated")
+
+        if extract_images:
+            log("Images Extracted")
+
+        if chunk_size is not None:
+            chunks = chunk_document(document, chunk_size=chunk_size, overlap=chunk_overlap)
+            write_chunks(chunks, out_dir / "chunks.json")
+            log(f"Chunks Generated ({len(chunks)} chunks)")
+
+        console.print(
+            f"\n[bold green]Done.[/] Output written to [cyan]{out_dir.resolve()}[/]"
+        )
+
+    except PDFConversionError as exc:
+        console.print(f"[bold red]Error:[/bold red] {exc}")
+        raise typer.Exit(code=1) from exc
+    except Exception as exc:
+        console.print(f"[bold red]Unexpected error:[/bold red] {exc}")
+        if verbose:
+            console.print_exception()
+        raise typer.Exit(code=1) from exc
+
+
+def _dispatch(argv: list[str] | None = None) -> None:
+    """Route argv to subcommands or the convert command."""
+    args = list(argv if argv is not None else sys.argv[1:])
+
+    if not args:
+        app(args=[])
+        return
+
+    head = args[0]
+    if head in _SUBCOMMANDS:
+        app(args=args)
+        return
+
+    if head in ("--help", "-h") and len(args) == 1:
+        app(args=args)
+        return
+
+    convert_app(args=args, prog_name="pdf2mj")
+
+
+def run() -> None:
+    """Console script entry point."""
+    from pdf2mj.console_util import ensure_utf8_stdio
+
+    ensure_utf8_stdio()
+    _dispatch()
+
+
+if __name__ == "__main__":
+    run()

pdf2mj/config.py

@@ -0,0 +1,70 @@
+"""User configuration and onboarding state."""
+
+from __future__ import annotations
+
+import json
+from pathlib import Path
+
+from platformdirs import user_config_dir
+from pydantic import BaseModel
+
+APP_NAME = "pdf2mj"
+CONFIG_FILENAME = "config.json"
+
+
+class UserConfig(BaseModel):
+    """Persisted user preferences and onboarding flags."""
+
+    welcome_shown: bool = False
+
+
+def get_config_dir() -> Path:
+    """Return the platform-specific config directory for PDF2MJ."""
+    return Path(user_config_dir(APP_NAME, appauthor=False))
+
+
+def get_config_path() -> Path:
+    """Return the path to the user config file."""
+    return get_config_dir() / CONFIG_FILENAME
+
+
+def load_config() -> UserConfig:
+    """Load config from disk, returning defaults if missing or invalid."""
+    path = get_config_path()
+    if not path.exists():
+        return UserConfig()
+
+    try:
+        data = json.loads(path.read_text(encoding="utf-8"))
+        return UserConfig.model_validate(data)
+    except (json.JSONDecodeError, OSError, ValueError):
+        return UserConfig()
+
+
+def save_config(config: UserConfig) -> None:
+    """Persist config to disk."""
+    path = get_config_path()
+    path.parent.mkdir(parents=True, exist_ok=True)
+    path.write_text(
+        json.dumps(config.model_dump(), indent=2) + "\n",
+        encoding="utf-8",
+    )
+
+
+def is_welcome_shown() -> bool:
+    """Return whether the first-run welcome screen has been displayed."""
+    return load_config().welcome_shown
+
+
+def mark_welcome_shown() -> None:
+    """Mark the welcome screen as shown."""
+    config = load_config()
+    config.welcome_shown = True
+    save_config(config)
+
+
+def reset_welcome_shown() -> None:
+    """Reset onboarding state (primarily for tests)."""
+    config = load_config()
+    config.welcome_shown = False
+    save_config(config)

pdf2mj/console_util.py

@@ -0,0 +1,26 @@
+"""Shared Rich console helpers."""
+
+from __future__ import annotations
+
+import sys
+
+from rich.console import Console
+
+
+def ensure_utf8_stdio() -> None:
+    """Reconfigure stdout/stderr to UTF-8 on Windows when supported."""
+    if sys.platform != "win32":
+        return
+    for stream in (sys.stdout, sys.stderr):
+        reconfigure = getattr(stream, "reconfigure", None)
+        if callable(reconfigure):
+            try:
+                reconfigure(encoding="utf-8")
+            except Exception:
+                pass
+
+
+def make_console() -> Console:
+    """Create a Rich console with sensible cross-platform defaults."""
+    ensure_utf8_stdio()
+    return Console(legacy_windows=False)

pdf2mj/converter.py

@@ -0,0 +1,197 @@
+"""PDF parsing pipeline that builds the common Document model."""
+
+from __future__ import annotations
+
+import re
+from pathlib import Path
+
+import fitz
+
+from pdf2mj.image_extractor import extract_images_from_page
+from pdf2mj.metadata import extract_metadata
+from pdf2mj.models import Block, Document, Page
+from pdf2mj.ocr import ocr_page
+from pdf2mj.table_extractor import extract_tables_from_page
+
+
+class PDFConversionError(Exception):
+    """Raised when a PDF cannot be processed."""
+
+
+def _classify_block(
+    text: str,
+    font_size: float,
+    body_size: float,
+    is_bold: bool,
+    is_mono: bool,
+    is_italic: bool,
+) -> tuple[str, int | None]:
+    stripped = text.strip()
+    if not stripped:
+        return "paragraph", None
+
+    if is_mono or re.match(r"^(\s{4}|\t)", stripped):
+        return "code", None
+
+    if stripped.startswith(">") or (is_italic and len(stripped) < 200):
+        return "quote", None
+
+    list_pattern = re.compile(
+        r"^(\u2022|\u2023|\u25E6|\u2043|\u2219|[-*•]|\d+[.)])\s+",
+        re.MULTILINE,
+    )
+    if list_pattern.match(stripped) or "\n" in stripped and all(
+        list_pattern.match(line.strip()) or not line.strip()
+        for line in stripped.splitlines()
+        if line.strip()
+    ):
+        return "list", None
+
+    if font_size >= body_size * 1.35 or (is_bold and font_size >= body_size * 1.15):
+        if font_size >= body_size * 1.8:
+            return "heading", 1
+        if font_size >= body_size * 1.5:
+            return "heading", 2
+        return "heading", 3
+
+    return "paragraph", None
+
+
+def _parse_text_blocks(page: fitz.Page, block_id_start: int) -> tuple[list[Block], int, bool]:
+    """Parse text blocks from a page; returns (blocks, next_id, has_text)."""
+    blocks: list[Block] = []
+    next_id = block_id_start
+    has_text = False
+
+    data = page.get_text("dict", flags=fitz.TEXT_PRESERVE_WHITESPACE)
+    font_sizes: list[float] = []
+
+    raw_blocks: list[dict] = []
+    for b in data.get("blocks", []):
+        if b.get("type") != 0:
+            continue
+        spans_info: list[dict] = []
+        full_text = ""
+        for line in b.get("lines", []):
+            for span in line.get("spans", []):
+                t = span.get("text", "")
+                full_text += t
+                size = span.get("size", 12.0)
+                font_sizes.append(size)
+                spans_info.append(
+                    {
+                        "size": size,
+                        "flags": span.get("flags", 0),
+                        "font": span.get("font", ""),
+                    }
+                )
+
+        text = full_text.strip()
+        if not text:
+            continue
+
+        raw_blocks.append(
+            {
+                "text": text,
+                "bbox": list(b.get("bbox", [])),
+                "spans": spans_info,
+            }
+        )
+
+    body_size = sum(font_sizes) / len(font_sizes) if font_sizes else 12.0
+
+    for rb in raw_blocks:
+        has_text = True
+        spans = rb["spans"]
+        avg_size = sum(s["size"] for s in spans) / len(spans) if spans else body_size
+        flags = spans[0]["flags"] if spans else 0
+        font_name = spans[0]["font"].lower() if spans else ""
+        is_bold = bool(flags & 2**4)
+        is_italic = bool(flags & 2**1)
+        is_mono = "mono" in font_name or "courier" in font_name or "consolas" in font_name
+
+        block_type, level = _classify_block(
+            rb["text"], avg_size, body_size, is_bold, is_mono, is_italic
+        )
+        blocks.append(
+            Block(
+                id=f"blk_{next_id}",
+                type=block_type,
+                text=rb["text"],
+                bbox=rb["bbox"],
+                level=level,
+            )
+        )
+        next_id += 1
+
+    return blocks, next_id, has_text
+
+
+def parse_pdf(
+    pdf_path: Path,
+    *,
+    use_ocr: bool = False,
+    extract_images: bool = False,
+    extract_tables: bool = True,
+    images_output_dir: Path | None = None,
+) -> Document:
+    """Parse a PDF file into the common Document model."""
+    if not pdf_path.exists():
+        raise PDFConversionError(f"File not found: {pdf_path}")
+
+    try:
+        doc = fitz.open(pdf_path)
+    except Exception as exc:
+        raise PDFConversionError(f"Cannot open PDF (corrupt or invalid): {pdf_path.name}") from exc
+
+    if doc.is_encrypted:
+        doc.close()
+        raise PDFConversionError(
+            f"PDF is encrypted and cannot be read without a password: {pdf_path.name}"
+        )
+
+    if doc.page_count == 0:
+        doc.close()
+        raise PDFConversionError(f"PDF has no pages: {pdf_path.name}")
+
+    metadata = extract_metadata(doc)
+    pages: list[Page] = []
+    global_block_id = 1
+
+    for page_index in range(doc.page_count):
+        page = doc[page_index]
+        page_number = page_index + 1
+        page_blocks: list[Block] = []
+
+        table_blocks: list[Block] = []
+        if extract_tables:
+            table_blocks, global_block_id = extract_tables_from_page(
+                page, page_number, global_block_id
+            )
+
+        text_blocks, global_block_id, has_text = _parse_text_blocks(page, global_block_id)
+
+        if use_ocr and not has_text:
+            ocr_blocks, global_block_id = ocr_page(page, global_block_id)
+            text_blocks = ocr_blocks
+
+        if extract_images and images_output_dir is not None:
+            img_blocks, global_block_id = extract_images_from_page(
+                page, page_number, images_output_dir, global_block_id
+            )
+            page_blocks.extend(img_blocks)
+
+        page_blocks.extend(text_blocks)
+        page_blocks.extend(table_blocks)
+
+        pages.append(Page(page_number=page_number, blocks=page_blocks))
+
+    page_count = doc.page_count
+    doc.close()
+
+    return Document(
+        file_name=pdf_path.name,
+        page_count=page_count,
+        metadata=metadata,
+        pages=pages,
+    )