parsearena 0.1.3__py3-none-any.whl

parsearena/errors.py

@@ -0,0 +1,14 @@
+"""Errors that are shown to the user as a short message, never as a traceback."""
+
+from __future__ import annotations
+
+
+class UserError(Exception):
+    def __init__(self, message: str, hint: str = "") -> None:
+        super().__init__(message)
+        self.message = message
+        self.hint = hint
+
+
+class BudgetExceeded(UserError):
+    pass

parsearena/evaluate.py

@@ -0,0 +1,130 @@
+"""Turn parser outputs plus a question set into retrieval metrics."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+
+from .retrieval import BM25, Embedder, rrf, score_ranks
+from .text import chunk_text, contains, normalize
+
+MRR_CUTOFF = 10
+
+
+@dataclass
+class Question:
+    id: str
+    doc: str
+    q: str
+    answers: list[str]
+
+
+@dataclass
+class DocOutput:
+    pages: list[str]
+    seconds: float = 0.0
+    error: str | None = None
+    device: str = ""
+    warmup_seconds: float | None = None
+    extra: dict = field(default_factory=dict)
+
+    @property
+    def text(self) -> str:
+        return "\n\n".join(self.pages)
+
+    @property
+    def ok(self) -> bool:
+        return self.error is None
+
+
+def _first_hit(ranks: dict[int, int], relevant: set[int]) -> int | None:
+    hits = [ranks[i] for i in relevant if i in ranks]
+    return min(hits) if hits else None
+
+
+def evaluate(
+    outputs: dict[str, dict[str, DocOutput]],
+    questions: list[Question],
+    k: int = 5,
+    embedder: Embedder | None = None,
+    chunk_chars: int = 1200,
+    progress=None,
+) -> dict:
+    """Score every parser on every question.
+
+    Retrieval runs over the chunks of *all* documents produced by one parser, so
+    a question can be answered from the wrong document. A chunk is relevant when
+    it belongs to the question's document and contains one of the answers.
+    """
+    methods = ["bm25"] + (["dense", "hybrid"] if embedder is not None else [])
+    q_vecs = embedder.embed_queries([q.q for q in questions]) if (embedder and questions) else None
+    result: dict[str, dict] = {}
+    for parser, docs in outputs.items():
+        if progress:
+            progress(parser)
+        chunks = []
+        for doc_id, out in docs.items():
+            if out.ok:
+                chunks.extend(chunk_text(doc_id, out.text, chunk_chars))
+        texts = [c.text for c in chunks]
+        norm_chunks = [normalize(t) for t in texts]
+        norm_docs = {d: normalize(o.text) for d, o in docs.items() if o.ok}
+        bm25 = BM25(texts) if texts else None
+        c_vecs = embedder.embed(texts) if (embedder and texts) else None
+        per_q = []
+        for qi, q in enumerate(questions):
+            survives = any(contains(norm_docs.get(q.doc, ""), a) for a in q.answers)
+            relevant = {
+                i
+                for i, c in enumerate(chunks)
+                if c.doc == q.doc and any(contains(norm_chunks[i], a) for a in q.answers)
+            }
+            ranks: dict[str, int | None] = {}
+            if texts:
+                orders = {"bm25": score_ranks(bm25.scores(q.q), drop_zero=True)}
+                if c_vecs is not None and q_vecs is not None:
+                    orders["dense"] = score_ranks((c_vecs @ q_vecs[qi]).tolist())
+                    fused = rrf([orders["bm25"], orders["dense"]])
+                    orders["hybrid"] = score_ranks([fused.get(i, 0.0) for i in range(len(texts))])
+                for m in methods:
+                    ranks[m] = _first_hit(orders[m], relevant)
+            else:
+                ranks = {m: None for m in methods}
+            per_q.append({"id": q.id, "doc": q.doc, "survives": survives, "ranks": ranks})
+        summary = {"survives": _mean([p["survives"] for p in per_q]), "chunks": len(chunks)}
+        for m in methods:
+            rs = [p["ranks"][m] for p in per_q]
+            summary[m] = {
+                "recall": _mean([r is not None and r <= k for r in rs]),
+                "mrr": _mean([(1.0 / r) if (r is not None and r <= MRR_CUTOFF) else 0.0 for r in rs]),
+            }
+        result[parser] = {"summary": summary, "questions": per_q}
+    return {
+        "k": k,
+        "mrr_cutoff": MRR_CUTOFF,
+        "methods": methods,
+        "chunk_chars": chunk_chars,
+        "parsers": result,
+    }
+
+
+def timings(outputs: dict[str, dict[str, DocOutput]], pages: dict[str, int]) -> dict[str, dict]:
+    out = {}
+    for parser, docs in outputs.items():
+        ok = [(d, o) for d, o in docs.items() if o.ok]
+        n_pages = sum(pages.get(d, len(o.pages)) for d, o in ok)
+        secs = sum(o.seconds for _, o in ok)
+        warm = next((o.warmup_seconds for o in docs.values() if o.warmup_seconds is not None), None)
+        out[parser] = {
+            "pages": n_pages,
+            "seconds": secs,
+            "seconds_per_page": (secs / n_pages) if n_pages else None,
+            "warmup_seconds": warm,
+            "failed": [{"doc": d, "error": o.error} for d, o in docs.items() if not o.ok],
+            "device": next((o.device for o in docs.values() if o.device), ""),
+        }
+    return out
+
+
+def _mean(values) -> float:
+    values = list(values)
+    return (sum(float(v) for v in values) / len(values)) if values else 0.0

parsearena/llm.py

@@ -0,0 +1,154 @@
+"""One OpenAI-compatible client for every provider, with a hard spending cap."""
+
+from __future__ import annotations
+
+import os
+import time
+from dataclasses import dataclass, field
+
+from .errors import BudgetExceeded, UserError
+
+
+@dataclass(frozen=True)
+class Preset:
+    base_url: str
+    key_env: str | None
+    model: str
+    price_in: float  # USD per million input tokens (estimate used for the pre-call check)
+    price_out: float  # USD per million output tokens
+
+
+PRESETS: dict[str, Preset] = {
+    "ollama": Preset("http://localhost:11434/v1", None, "qwen3:4b", 0.0, 0.0),
+    "openrouter": Preset(
+        "https://openrouter.ai/api/v1", "OPENROUTER_API_KEY", "deepseek/deepseek-v4.1-flash", 0.045, 0.50
+    ),
+    "openai": Preset("https://api.openai.com/v1", "OPENAI_API_KEY", "gpt-5.6-luna", 0.20, 1.20),
+    "mistral": Preset("https://api.mistral.ai/v1", "MISTRAL_API_KEY", "ministral-8b-latest", 0.10, 0.10),
+}
+
+
+def ollama_base_url() -> str:
+    host = os.environ.get("OLLAMA_HOST", "").strip()
+    if not host:
+        return PRESETS["ollama"].base_url
+    if not host.startswith(("http://", "https://")):
+        host = f"http://{host}"
+    return host.rstrip("/") + "/v1"
+
+
+@dataclass
+class LLM:
+    provider: str
+    model: str
+    max_cost: float = 1.0
+    price_in: float | None = None
+    price_out: float | None = None
+    spent: float = 0.0
+    calls: int = 0
+    _client: object = field(default=None, repr=False)
+
+    def __post_init__(self) -> None:
+        if self.provider not in PRESETS:
+            raise UserError(f"Unknown provider: {self.provider}", f"Choose one of: {', '.join(PRESETS)}")
+        preset = PRESETS[self.provider]
+        self.model = self.model or preset.model
+        if self.price_in is None:
+            self.price_in = preset.price_in
+        if self.price_out is None:
+            self.price_out = preset.price_out
+        if preset.key_env:
+            key = os.environ.get(preset.key_env, "").strip()
+            if not key:
+                raise UserError(
+                    f"Missing API key: set {preset.key_env}.",
+                    f"Example: export {preset.key_env}=...  or use --provider ollama for a free local model.",
+                )
+            base_url = preset.base_url
+        else:
+            key, base_url = "ollama", ollama_base_url()
+        from openai import OpenAI
+
+        self.base_url = base_url
+        self.timeout = float(os.environ.get("PARSEARENA_LLM_TIMEOUT", "300"))
+        self._client = OpenAI(base_url=base_url, api_key=key, max_retries=0, timeout=self.timeout)
+
+    @property
+    def local(self) -> bool:
+        return self.provider == "ollama"
+
+    def estimate(self, prompt: str, max_tokens: int) -> float:
+        tokens_in = len(prompt) / 3.5
+        return (tokens_in * self.price_in + max_tokens * self.price_out) / 1e6
+
+    def complete(self, system: str, prompt: str, max_tokens: int = 2000) -> str:
+        if not self.local:
+            projected = self.spent + self.estimate(system + prompt, max_tokens)
+            if projected > self.max_cost:
+                raise BudgetExceeded(
+                    f"Budget exceeded: the next call could bring spend to ${projected:.4f}, "
+                    f"over --max-cost ${self.max_cost:.2f}.",
+                    "Partial results were kept. Raise --max-cost or use --provider ollama.",
+                )
+        extra = {"usage": {"include": True}} if self.provider == "openrouter" else None
+        resp = self._call(system, prompt, max_tokens, extra)
+        self.calls += 1
+        self.spent += self._cost(resp)
+        return (resp.choices[0].message.content or "").strip()
+
+    def _call(self, system, prompt, max_tokens, extra):
+        import openai
+
+        delay = 2.0
+        for attempt in range(5):
+            try:
+                return self._client.chat.completions.create(
+                    model=self.model,
+                    messages=[{"role": "system", "content": system}, {"role": "user", "content": prompt}],
+                    max_tokens=max_tokens,
+                    temperature=0.2,
+                    extra_body=extra,
+                )
+            except openai.APITimeoutError as exc:
+                if self.local or attempt == 4:
+                    raise UserError(
+                        f"{self.provider} did not answer within {self.timeout:.0f}s.",
+                        "The server may be busy or still loading the model. Try again, "
+                        "or raise the limit with PARSEARENA_LLM_TIMEOUT (seconds).",
+                    ) from exc
+            except openai.APIConnectionError as exc:
+                if self.local:
+                    raise UserError(
+                        f"Ollama is not reachable at {self.base_url}.",
+                        "Start it with `ollama serve`, or set OLLAMA_HOST.",
+                    ) from exc
+                if attempt == 4:
+                    raise UserError(f"Could not reach {self.provider}: {exc}") from exc
+            except openai.AuthenticationError as exc:
+                raise UserError(
+                    f"{self.provider} rejected the API key.", f"Check {PRESETS[self.provider].key_env}."
+                ) from exc
+            except openai.NotFoundError as exc:
+                hint = f"Pull it with `ollama pull {self.model}`." if self.local else "Check --model."
+                raise UserError(f"Model not found on {self.provider}: {self.model}", hint) from exc
+            except (openai.RateLimitError, openai.InternalServerError) as exc:
+                if attempt == 4:
+                    raise UserError(f"{self.provider} keeps failing: {exc}", "Try again later.") from exc
+            time.sleep(delay)
+            delay *= 2
+        raise UserError(f"{self.provider} did not answer.")
+
+    def _cost(self, resp) -> float:
+        if self.local:
+            return 0.0
+        usage = getattr(resp, "usage", None)
+        if usage is None:
+            return 0.0
+        reported = getattr(usage, "cost", None)
+        if reported is None and getattr(usage, "model_extra", None):
+            reported = usage.model_extra.get("cost")
+        if reported is not None:
+            return float(reported)
+        return (
+            (usage.prompt_tokens or 0) * self.price_in + (usage.completion_tokens or 0) * self.price_out
+        ) / 1e6

parsearena/parsers.py

@@ -0,0 +1,197 @@
+"""Parser registry and isolated environments.
+
+Every parser lives in its own uv-managed virtual environment under
+``$PARSEARENA_HOME/envs`` and is driven through a small runner script in a
+subprocess. parsearena never imports a parser and never bundles one.
+"""
+
+from __future__ import annotations
+
+import hashlib
+import json
+import os
+import shutil
+import subprocess
+import sys
+from dataclasses import dataclass
+from importlib import resources
+from pathlib import Path
+
+from .errors import UserError
+
+
+@dataclass(frozen=True)
+class ParserSpec:
+    name: str
+    label: str
+    packages: tuple[str, ...]
+    runner: str
+    license: str
+    url: str
+    python: str = "3.12"
+    env: tuple[tuple[str, str], ...] = ()
+    # (binary, env var that may point to it, install hint) checked before running
+    tools: tuple[tuple[str, str, str], ...] = ()
+    note: str = ""
+
+    @property
+    def version(self) -> str:
+        return self.packages[0].split("==", 1)[1]
+
+    @property
+    def key(self) -> str:
+        digest = hashlib.sha256(json.dumps([self.packages, self.python]).encode()).hexdigest()[:8]
+        return f"{self.name}-{self.version}-py{self.python.replace('.', '')}-{digest}"
+
+
+PARSERS: dict[str, ParserSpec] = {
+    "docling": ParserSpec(
+        name="docling",
+        label="Docling",
+        packages=("docling==2.146.0",),
+        runner="run_docling.py",
+        license="MIT",
+        url="https://github.com/docling-project/docling",
+    ),
+    "marker": ParserSpec(
+        name="marker",
+        label="Marker",
+        packages=("marker-pdf==2.1.0",),
+        runner="run_marker.py",
+        license="Code Apache-2.0; model weights modified OpenRAIL-M",
+        url="https://github.com/datalab-to/marker",
+        tools=(
+            (
+                "llama-server",
+                "LLAMA_CPP_BINARY",
+                "Marker 2.x runs its OCR model through llama.cpp when there is no NVIDIA GPU: "
+                "`brew install llama.cpp`, or set LLAMA_CPP_BINARY to a llama-server binary.",
+            ),
+        ),
+    ),
+    "mineru": ParserSpec(
+        name="mineru",
+        label="MinerU",
+        packages=("mineru==4.1.6",),
+        runner="run_mineru.py",
+        license="Apache-2.0 with additional terms",
+        url="https://github.com/opendatalab/MinerU",
+        env=(("MINERU_MODEL_SOURCE", "huggingface"), ("MINERU_TELEMETRY_DISABLED", "1")),
+        note='effort "medium": local layout, OCR, table and formula models, no VLM',
+    ),
+    "unstructured": ParserSpec(
+        name="unstructured",
+        label="Unstructured",
+        packages=("unstructured[pdf]==0.27.36",),
+        runner="run_unstructured.py",
+        license="Apache-2.0",
+        url="https://github.com/Unstructured-IO/unstructured",
+        tools=(
+            (
+                "pdftoppm",
+                "",
+                "Unstructured hi_res needs poppler: `brew install poppler` / `apt install poppler-utils`.",
+            ),
+            (
+                "tesseract",
+                "",
+                "Unstructured hi_res needs tesseract: "
+                "`brew install tesseract` / `apt install tesseract-ocr`.",
+            ),
+        ),
+    ),
+    "pypdf": ParserSpec(
+        name="pypdf",
+        label="pypdf (baseline)",
+        packages=("pypdf==6.21.4",),
+        runner="run_pypdf.py",
+        license="BSD-3-Clause",
+        url="https://github.com/py-pdf/pypdf",
+    ),
+}
+
+DEFAULT_PARSERS = ("docling", "marker", "mineru", "unstructured", "pypdf")
+
+
+def parse_parser_list(value: str) -> list[ParserSpec]:
+    names = [n.strip().lower() for n in value.split(",") if n.strip()]
+    if not names:
+        raise UserError("No parsers selected.", f"Choose from: {', '.join(PARSERS)}")
+    unknown = [n for n in names if n not in PARSERS]
+    if unknown:
+        raise UserError(
+            f"Unknown parser: {', '.join(unknown)}",
+            f"Available parsers: {', '.join(PARSERS)}",
+        )
+    return [PARSERS[n] for n in dict.fromkeys(names)]
+
+
+def home() -> Path:
+    if os.environ.get("PARSEARENA_HOME"):
+        return Path(os.environ["PARSEARENA_HOME"]).expanduser()
+    base = os.environ.get("XDG_CACHE_HOME") or Path.home() / ".cache"
+    return Path(base) / "parsearena"
+
+
+def env_dir(spec: ParserSpec) -> Path:
+    return home() / "envs" / spec.key
+
+
+def env_python(spec: ParserSpec) -> Path:
+    d = env_dir(spec)
+    return d / ("Scripts/python.exe" if sys.platform == "win32" else "bin/python")
+
+
+def missing_tools(spec: ParserSpec) -> list[str]:
+    """Install hints for system tools the parser needs but that are not available."""
+    hints = []
+    for binary, env_var, hint in spec.tools:
+        configured = os.environ.get(env_var) if env_var else None
+        if configured and Path(configured).exists():
+            continue
+        if shutil.which(binary) is None:
+            hints.append(hint)
+    return hints
+
+
+def find_uv() -> str:
+    uv = shutil.which("uv")
+    if uv is None:
+        raise UserError(
+            "uv was not found on PATH.",
+            "parsearena installs each parser in its own uv environment. "
+            "Install uv: https://docs.astral.sh/uv/getting-started/installation/",
+        )
+    return uv
+
+
+def is_installed(spec: ParserSpec) -> bool:
+    return (env_dir(spec) / ".parsearena-ok").exists() and env_python(spec).exists()
+
+
+def ensure_env(spec: ParserSpec, log=print) -> Path:
+    """Create the parser's environment if needed and return its python."""
+    if is_installed(spec):
+        return env_python(spec)
+    uv = find_uv()
+    d = env_dir(spec)
+    d.parent.mkdir(parents=True, exist_ok=True)
+    log(f"Installing {spec.label} ({', '.join(spec.packages)}) into {d}")
+    steps = [
+        [uv, "venv", "--quiet", "--clear", "--python", spec.python, str(d)],
+        [uv, "pip", "install", "--quiet", "--python", str(env_python(spec)), *spec.packages],
+    ]
+    for cmd in steps:
+        proc = subprocess.run(cmd, capture_output=True, text=True)
+        if proc.returncode != 0:
+            tail = (proc.stderr or proc.stdout).strip().splitlines()[-15:]
+            raise UserError(
+                f"Could not install {spec.label}.",
+                "uv said:\n" + "\n".join(tail),
+            )
+    (d / ".parsearena-ok").write_text(json.dumps({"packages": spec.packages}))
+    return env_python(spec)
+
+
+def runner_dir() -> Path:
+    return Path(str(resources.files("parsearena") / "runners"))