parsearena 0.1.3__py3-none-any.whl

parsearena/__init__.py

@@ -0,0 +1,3 @@
+"""parsearena: compare PDF parsers on your own documents and measure retrieval."""
+
+__version__ = "0.1.3"

parsearena/cli.py

@@ -0,0 +1,288 @@
+"""Command line interface."""
+
+from __future__ import annotations
+
+import json
+import sys
+import time
+import webbrowser
+from importlib import resources
+from pathlib import Path
+
+import typer
+from rich.console import Console
+from rich.table import Table
+
+from . import __version__
+from .errors import UserError
+from .evaluate import evaluate, timings
+from .parsers import DEFAULT_PARSERS, PARSERS, env_dir, home, is_installed, missing_tools, parse_parser_list
+from .pipeline import collect_docs, load_outputs, run_parser, save_outputs
+from .questions import EXAMPLE, dump_questions, generate_questions, load_questions
+from .retrieval import DEFAULT_EMBED_MODEL
+
+app = typer.Typer(
+    add_completion=False,
+    no_args_is_help=True,
+    rich_markup_mode=None,
+    help="Run your PDFs through Docling, Marker, MinerU and Unstructured. "
+    "Compare the output side by side and see what each one does to retrieval.",
+)
+console = Console(highlight=False)
+err = Console(stderr=True, highlight=False)
+
+
+def _version(value: bool) -> None:
+    if value:
+        console.print(f"parsearena {__version__}")
+        raise typer.Exit()
+
+
+@app.callback()
+def _root(
+    version: bool = typer.Option(False, "--version", callback=_version, is_eager=True, help="Show version."),
+) -> None:
+    pass
+
+
+def _embedder(no_dense: bool, model: str):
+    if no_dense:
+        return None
+    try:
+        from .retrieval import FastEmbedder, model_is_cached
+
+        cache = str(home() / "models")
+        if model_is_cached(model, cache):
+            msg = f"Loading embedding model {model}..."
+        else:
+            size = ", about 65 MB" if model == DEFAULT_EMBED_MODEL else ""
+            msg = f"Downloading embedding model {model} (one time{size})..."
+            console.print(msg)
+        with console.status(msg):
+            return FastEmbedder(model, cache_dir=cache)
+    except Exception as exc:  # network errors, unknown model names
+        raise UserError(
+            f"Could not load the embedding model {model}: {type(exc).__name__}: {exc}"[:400],
+            "Check your connection for the first download, pick another --embed-model, "
+            "or use --no-dense for BM25 only.",
+        ) from exc
+
+
+def _analyze(
+    recorded, pdf_paths, questions, out, k, no_dense, embed_model, chunk_chars, is_recorded, doc_info=None
+):
+    from .report import build_report
+
+    outputs = load_outputs(recorded)
+    embedder = _embedder(no_dense, embed_model)
+    with console.status("Computing retrieval metrics..."):
+        ev = evaluate(outputs, questions, k=k, embedder=embedder, chunk_chars=chunk_chars)
+    ev["embed_model"] = embedder.name if embedder else None
+    ev["questions"] = len(questions)
+    tm = timings(outputs, {d["id"]: d["num_pages"] for d in recorded["docs"]})
+    with console.status("Rendering pages and writing the viewer..."):
+        index = build_report(out, recorded, pdf_paths, questions, ev, tm, is_recorded, doc_info)
+    _print_summary(recorded, ev, tm)
+    return index
+
+
+def _pct(x: float) -> str:
+    # Round half up, like the viewer (Math.round), so terminal and HTML agree.
+    return f"{int(x * 100 + 0.5)}%"
+
+
+def _print_summary(recorded, ev, tm) -> None:
+    labels = {p["name"]: p["label"] for p in recorded["parsers"]}
+    k = ev["k"]
+    table = Table(title=None, header_style="bold", pad_edge=False)
+    table.add_column("parser")
+    table.add_column("survives", justify="right")
+    for m in ev["methods"]:
+        table.add_column(f"R@{k} {m}", justify="right")
+    table.add_column("MRR " + ev["methods"][-1], justify="right")
+    table.add_column("s/page", justify="right")
+    table.add_column("failed", justify="right")
+    has_q = ev.get("questions", 0) > 0
+    for p, res in ev["parsers"].items():
+        s, t = res["summary"], tm[p]
+        spp = t["seconds_per_page"]
+        row = [labels.get(p, p), _pct(s["survives"]) if has_q else "-"]
+        row += [_pct(s[m]["recall"]) if has_q else "-" for m in ev["methods"]]
+        row += [f"{s[ev['methods'][-1]]['mrr']:.2f}" if has_q else "-"]
+        row += ["-" if spp is None else (f"{spp:.3f}" if spp < 0.1 else f"{spp:.1f}")]
+        row += [str(len(t["failed"])) if t["failed"] else "0"]
+        table.add_row(*row)
+    console.print(table)
+    if not has_q:
+        console.print(
+            "[yellow]No questions given, so retrieval metrics are empty.[/] "
+            "Pass --questions FILE or --gen-questions."
+        )
+
+
+def _open(index: Path, no_open: bool) -> None:
+    console.print(f"\nViewer: [bold]{index}[/]")
+    if not no_open:
+        webbrowser.open(index.resolve().as_uri())
+
+
+@app.command()
+def run(
+    pdfs: list[Path] = typer.Argument(..., help="PDF files to parse.", show_default=False),
+    parsers: str = typer.Option(
+        ",".join(DEFAULT_PARSERS), "--parsers", "-p", help="Comma-separated parsers."
+    ),
+    questions: Path | None = typer.Option(None, "--questions", "-q", help="YAML file with questions."),
+    gen_questions: bool = typer.Option(False, "--gen-questions", help="Draft questions with an LLM."),
+    per_doc: int = typer.Option(5, "--per-doc", help="Questions to draft per document."),
+    provider: str = typer.Option("ollama", "--provider", help="ollama, openrouter, openai or mistral."),
+    model: str = typer.Option("", "--model", help="Model for --gen-questions (default depends on provider)."),
+    max_cost: float = typer.Option(1.0, "--max-cost", help="Spending cap in USD for LLM calls."),
+    out: Path = typer.Option(Path("parsearena-out"), "--out", "-o", help="Output directory."),
+    k: int = typer.Option(5, "--k", help="k for recall@k."),
+    device: str = typer.Option("auto", "--device", help="auto, cpu, mps or cuda."),
+    timeout: int = typer.Option(3600, "--timeout", help="Seconds allowed per parser."),
+    chunk_chars: int = typer.Option(1200, "--chunk-chars", help="Maximum characters per chunk."),
+    embed_model: str = typer.Option(DEFAULT_EMBED_MODEL, "--embed-model", help="fastembed model name."),
+    no_dense: bool = typer.Option(False, "--no-dense", help="BM25 only, no embeddings."),
+    no_open: bool = typer.Option(False, "--no-open", help="Do not open the viewer."),
+) -> None:
+    """Parse PDFs with each parser in its own environment, then score retrieval."""
+    specs = parse_parser_list(parsers)
+    if device not in ("auto", "cpu", "mps", "cuda"):
+        raise UserError(f"Unknown device: {device}", "Use auto, cpu, mps or cuda.")
+    docs = collect_docs(pdfs)
+    out.mkdir(parents=True, exist_ok=True)
+
+    qs = []
+    if questions:
+        qs = load_questions(questions, docs)
+    elif gen_questions:
+        from .llm import LLM
+        from .render import page_text
+
+        llm = LLM(provider=provider, model=model, max_cost=max_cost)
+        console.print(f"Drafting questions with {provider}:{llm.model}")
+        texts = {d["id"]: "\n\n".join(page_text(Path(d["path"]))) for d in docs}
+        qs, _, budget_err = generate_questions(llm, docs, texts, per_doc=per_doc, log=console.print)
+        qpath = out / "questions.generated.yaml"
+        dump_questions(
+            qs, docs, qpath, header=f"# Drafted by {provider}:{llm.model}. Review before trusting."
+        )
+        console.print(f"Saved {len(qs)} questions to {qpath} (spent ${llm.spent:.4f})")
+        if budget_err:
+            raise budget_err
+    else:
+        console.print(
+            "[yellow]No --questions or --gen-questions: only the side-by-side viewer and timings.[/]"
+        )
+
+    outputs = {}
+    work = out / "work"
+    for spec in specs:
+        console.rule(f"{spec.label} {spec.version}")
+        for hint in missing_tools(spec):
+            console.print(f"[yellow]Warning:[/] {hint}")
+        if not is_installed(spec):
+            console.print(
+                f"First use of {spec.label}: installing it in its own environment. "
+                "This and the first parse can download several GB."
+            )
+        t0 = time.perf_counter()
+        with console.status(f"Running {spec.label} on {len(docs)} PDF(s)..."):
+            outputs[spec.name] = run_parser(
+                spec, docs, work, device=device, timeout=timeout, log=console.print
+            )
+        failed = [d for d, o in outputs[spec.name].items() if not o.ok]
+        status = f"[red]{len(failed)} failed[/]" if failed else "[green]ok[/]"
+        console.print(f"{spec.label}: {status} in {time.perf_counter() - t0:.0f}s (log: {work / spec.name})")
+
+    recorded = save_outputs(out / "outputs.json", docs, specs, outputs, device)
+    pdf_paths = {d["id"]: Path(d["path"]) for d in docs}
+    index = _analyze(recorded, pdf_paths, qs, out, k, no_dense, embed_model, chunk_chars, is_recorded=False)
+    _open(index, no_open)
+
+
+@app.command()
+def demo(
+    out: Path = typer.Option(Path("parsearena-demo"), "--out", "-o", help="Where to write the viewer."),
+    k: int = typer.Option(5, "--k", help="k for recall@k."),
+    no_dense: bool = typer.Option(False, "--no-dense", help="BM25 only, no embeddings download."),
+    no_open: bool = typer.Option(False, "--no-open", help="Do not open the viewer."),
+) -> None:
+    """Recorded parser outputs on 3 public PDFs; retrieval metrics computed live."""
+    base = Path(str(resources.files("parsearena") / "demo"))
+    recorded = json.loads((base / "recorded.json").read_text(encoding="utf-8"))
+    m = recorded["meta"]
+    created = m["created"].replace("T", " ").replace("Z", " UTC")
+    versions = ", ".join(f"{p['label']} {p['version']}" for p in recorded["parsers"])
+    console.print(
+        f"[bold]Recorded run[/] from {created} (parsearena {m['tool_version']}, {m['hardware']}).\n"
+        f"Parsers: {versions}.\nParser outputs are replayed; retrieval is computed now.\n"
+    )
+    docs = recorded["docs"]
+    qs = load_questions(base / "questions.yaml", docs)
+    pdf_paths = {d["id"]: base / "pdfs" / d["file"] for d in docs}
+    index = _analyze(
+        recorded,
+        pdf_paths,
+        qs,
+        out,
+        k,
+        no_dense,
+        DEFAULT_EMBED_MODEL,
+        recorded.get("chunk_chars", 1200),
+        is_recorded=True,
+    )
+    _open(index, no_open)
+
+
+@app.command("parsers")
+def list_parsers() -> None:
+    """List supported parsers, pinned versions, licenses and install state."""
+    table = Table(header_style="bold")
+    for col in ("name", "package", "license", "installed"):
+        table.add_column(col)
+    for spec in PARSERS.values():
+        table.add_row(
+            spec.name,
+            " ".join(spec.packages),
+            spec.license,
+            str(env_dir(spec)) if is_installed(spec) else "no",
+        )
+    console.print(table)
+    console.print(f"Environments live in {home() / 'envs'} (set PARSEARENA_HOME to move them).")
+
+
+@app.command("questions-example")
+def questions_example() -> None:
+    """Print an example questions YAML file."""
+    console.print(EXAMPLE, markup=False)
+
+
+def main() -> None:
+    try:
+        app(standalone_mode=False)
+    except UserError as exc:
+        err.print(f"[bold red]Error:[/] {exc.message}", markup=True)
+        if exc.hint:
+            err.print(exc.hint, markup=False)
+        sys.exit(1)
+    except KeyboardInterrupt:
+        err.print("Interrupted.")
+        sys.exit(130)
+    except typer.Exit as exc:
+        sys.exit(exc.exit_code)
+    except typer.Abort:
+        sys.exit(1)
+    except Exception as exc:
+        import click
+
+        if isinstance(exc, click.ClickException):
+            exc.show()
+            sys.exit(exc.exit_code)
+        raise
+
+
+if __name__ == "__main__":
+    main()

parsearena/demo/pdfs/NOTICE.md

@@ -0,0 +1,36 @@
+# Third-party documents
+
+The PDFs in this directory are excerpts of third-party documents, bundled only as demo input.
+They are not covered by parsearena's MIT license; each keeps its own terms, listed below.
+
+## census-income-2023.pdf
+
+- Title: Income in the United States: 2023 (Current Population Reports, P60-282), September 2024
+- Authors: Gloria Guzman and Melissa Kollar, U.S. Census Bureau
+- Source: https://www.census.gov/library/publications/2024/demo/p60-282.html
+  (PDF: https://www2.census.gov/library/publications/2024/demo/p60-282.pdf)
+- License: public domain. Work of the U.S. federal government, not subject to copyright in the
+  United States (17 U.S.C. § 105).
+- Changes: only report pages 1, 2 and 15 (PDF pages 7, 8 and 21) are included, unmodified.
+
+## dpo-arxiv-2305.18290.pdf
+
+- Title: Direct Preference Optimization: Your Language Model is Secretly a Reward Model (arXiv:2305.18290v3)
+- Authors: Rafael Rafailov, Archit Sharma, Eric Mitchell, Stefano Ermon, Christopher D. Manning, Chelsea Finn
+- Source: https://arxiv.org/abs/2305.18290
+- License: Creative Commons Attribution 4.0 International (CC BY 4.0),
+  https://creativecommons.org/licenses/by/4.0/ , as declared on the arXiv abstract page.
+- Attribution: "Direct Preference Optimization: Your Language Model is Secretly a Reward Model" by
+  Rafael Rafailov, Archit Sharma, Eric Mitchell, Stefano Ermon, Christopher D. Manning and Chelsea Finn,
+  arXiv:2305.18290v3, licensed under CC BY 4.0.
+- Changes: only pages 1, 4, 5 and 9 are included; the pages themselves are unmodified.
+
+## naca-report-1135-scan.pdf
+
+- Title: Equations, Tables, and Charts for Compressible Flow (NACA Report 1135), 1953
+- Authors: Ames Research Staff, Ames Aeronautical Laboratory, National Advisory Committee for Aeronautics
+- Source: https://ntrs.nasa.gov/citations/19930091059
+- License: public domain. Work of the U.S. federal government (17 U.S.C. § 105); NASA's Technical
+  Reports Server marks it "Public Use Permitted".
+- Changes: only report page 613 (PDF page 2) is included, re-rendered at 200 dpi in grayscale as an
+  image-only page so that it has no text layer, like a scan.

parsearena/demo/questions.yaml

@@ -0,0 +1,104 @@
+# Hand-written question set for the bundled demo PDFs.
+# Each answer is a short string copied from the original page.
+questions:
+  # U.S. Census Bureau, Income in the United States: 2023 (text page, chart page, table page)
+  - id: census-median
+    doc: census-income-2023.pdf
+    q: What was real median household income in the United States in 2023?
+    answers: ["80,610"]
+  - id: census-cpi
+    doc: census-income-2023.pdf
+    q: How much did consumer prices rise between 2022 and 2023 according to the C-CPI-U?
+    answers: ["4.0 percent increase"]
+  - id: census-earnings-ratio
+    doc: census-income-2023.pdf
+    q: What was the female-to-male earnings ratio for full-time, year-round workers in 2023?
+    answers: ["82.7 percent"]
+  - id: census-survey
+    doc: census-income-2023.pdf
+    q: Which survey supplement are the income estimates based on?
+    answers: ["Annual Social and Economic Supplements"]
+  - id: census-asian
+    doc: census-income-2023.pdf
+    q: What was the median income of Asian households in 2023?
+    answers: ["112,800"]
+  - id: census-bachelor
+    doc: census-income-2023.pdf
+    q: What was the median household income for householders with a bachelor's degree or higher in 2023?
+    answers: ["126,800"]
+  - id: census-midwest
+    doc: census-income-2023.pdf
+    q: What was the median household income in the Midwest in 2023?
+    answers: ["81,020"]
+  - id: census-nonmetro
+    doc: census-income-2023.pdf
+    q: What was the median income of households outside metropolitan statistical areas in 2023?
+    answers: ["62,520"]
+  - id: census-households
+    doc: census-income-2023.pdf
+    q: How many households (in thousands) were there in 2023?
+    answers: ["132,200"]
+  - id: census-foreign
+    doc: census-income-2023.pdf
+    q: What was the 2023 median income of households with a foreign-born householder?
+    answers: ["73,360"]
+
+  # Rafailov et al., Direct Preference Optimization (arXiv:2305.18290, CC BY 4.0)
+  - id: dpo-name
+    doc: dpo-arxiv-2305.18290.pdf
+    q: What does the acronym DPO stand for?
+    answers: ["Direct Preference Optimization"]
+  - id: dpo-preference-model
+    doc: dpo-arxiv-2305.18290.pdf
+    q: Under which preference model does the partition function cancel out in the DPO derivation?
+    answers: ["Bradley-Terry"]
+  - id: dpo-z
+    doc: dpo-arxiv-2305.18290.pdf
+    q: What is Z(x) called in the closed-form optimal policy?
+    answers: ["partition function"]
+  - id: dpo-gradient-weight
+    doc: dpo-arxiv-2305.18290.pdf
+    q: In the gradient of the DPO loss, when does an example get a higher weight?
+    answers: ["when reward estimate is wrong"]
+  - id: dpo-tldr
+    doc: dpo-arxiv-2305.18290.pdf
+    q: What win rate does DPO reach on TL;DR summarization at temperature 0.0?
+    answers: ["61%"]
+  - id: dpo-cnn
+    doc: dpo-arxiv-2305.18290.pdf
+    q: What GPT-4 win rate against ground truth does DPO get on CNN/DailyMail at temperature 0?
+    answers: ["0.36"]
+  - id: dpo-dialogue-model
+    doc: dpo-arxiv-2305.18290.pdf
+    q: Which pre-trained model is the starting point for the single-turn dialogue experiments?
+    answers: ["Pythia-2.8B"]
+  - id: dpo-venue
+    doc: dpo-arxiv-2305.18290.pdf
+    q: At which conference was the DPO paper published?
+    answers: ["NeurIPS 2023"]
+
+  # NACA Report 1135 (1953), a scanned page with no text layer (public domain)
+  - id: naca-report
+    doc: naca-report-1135-scan.pdf
+    q: What is the report number of "Equations, Tables, and Charts for Compressible Flow"?
+    answers: ["Report 1135"]
+  - id: naca-tn
+    doc: naca-report-1135-scan.pdf
+    q: Which NACA technical note does the compressible flow report revise and extend?
+    answers: ["1428"]
+  - id: naca-authors
+    doc: naca-report-1135-scan.pdf
+    q: Who is credited as the author of the compressible flow report?
+    answers: ["Ames Research Staff"]
+  - id: naca-gamma
+    doc: naca-report-1135-scan.pdf
+    q: What quantity does the symbol gamma stand for in the compressible flow notation?
+    answers: ["ratio of specific heats"]
+  - id: naca-extensions
+    doc: naca-report-1135-scan.pdf
+    q: In which direction were the compressible flow tables extended, according to the introduction?
+    answers: ["higher Mach numbers"]
+  - id: naca-vm
+    doc: naca-report-1135-scan.pdf
+    q: What does V_m denote in the list of symbols?
+    answers: ["maximum speed obtainable"]