pdfhell 0.1.0__py3-none-any.whl

pdfhell/junit.py

@@ -0,0 +1,94 @@
+"""Render a :class:`SuiteReport` as JUnit XML for CI consumption.
+
+GitHub Actions, GitLab CI, Jenkins, CircleCI, and most other CI runners
+display JUnit XML natively in their PR / merge-request panel. Failures
+show up as red rows on the PR with the model output and expected
+answer in the failure message.
+
+The rendered XML follows the de-facto JUnit dialect (Ant/Maven-style)
+that everyone parses. We classify outcomes as:
+
+- ``correct=True``    → passing testcase, no children
+- ``fell_for_trap``   → ``<failure>`` (this is the diagnostic signal)
+- ``refused``         → ``<skipped>`` (model wouldn't answer — not a quality fail)
+- everything else     → ``<failure>``
+
+We deliberately don't emit ``<error>`` — pdfhell's upstream errors
+(provider down, SDK missing) get caught in the runner and recorded as
+the model's text output starting with ``[error]``. Surfacing them as
+``<error>`` would noise the dashboard for transient infra issues.
+"""
+from __future__ import annotations
+
+import datetime
+import xml.etree.ElementTree as ET
+
+from .scorer import SuiteReport
+
+
+def _suite_name(report: SuiteReport) -> str:
+    """Slug-ish name for the JUnit suite — 'pdfhell.<suite>.<model>'."""
+    safe_model = report.model.replace("/", ".").replace(":", ".")
+    return f"pdfhell.{report.suite}.{safe_model}"
+
+
+def report_to_junit(report: SuiteReport) -> str:
+    """Return a JUnit XML string for ``report``.
+
+    The XML is human-readable and round-trips through every CI parser
+    we've tested. Failure messages include the expected and observed
+    answers so on-call doesn't have to dig through the runs JSON.
+    """
+    failures = sum(1 for c in report.cases if not c.correct and not c.refused)
+    skipped = sum(1 for c in report.cases if c.refused)
+    testsuite = ET.Element(
+        "testsuite",
+        {
+            "name": _suite_name(report),
+            "tests": str(report.n),
+            "failures": str(failures),
+            "errors": "0",
+            "skipped": str(skipped),
+            "timestamp": datetime.datetime.now(datetime.timezone.utc).isoformat(),
+        },
+    )
+    for c in report.cases:
+        case_el = ET.SubElement(
+            testsuite,
+            "testcase",
+            {
+                "name": c.case_id,
+                "classname": c.trap_family,
+            },
+        )
+        if c.refused:
+            skipped_el = ET.SubElement(case_el, "skipped", {"message": "model refused"})
+            skipped_el.text = c.model_output[:200]
+        elif not c.correct:
+            kind = "fell_for_trap" if c.fell_for_trap else "hallucination"
+            failure_el = ET.SubElement(
+                case_el,
+                "failure",
+                {
+                    "type": kind,
+                    "message": (
+                        f"expected={c.expected!r}; got={c.model_output[:80]!r}"
+                    ),
+                },
+            )
+            details = [
+                f"expected_answer: {c.expected}",
+                f"model_output:    {c.model_output}",
+            ]
+            if c.matched_forbidden:
+                details.append(f"matched_forbidden: {c.matched_forbidden}")
+            if c.failure_mode:
+                details.append(f"failure_mode: {c.failure_mode}")
+            failure_el.text = "\n".join(details)
+    testsuites = ET.Element("testsuites")
+    testsuites.append(testsuite)
+    # ET in 3.10+ supports short_empty_elements but xml_declaration is what CIs expect.
+    return ET.tostring(testsuites, encoding="unicode", xml_declaration=True)
+
+
+__all__ = ["report_to_junit"]

pdfhell/runner.py

@@ -0,0 +1,142 @@
+"""Run a model against a pdfhell suite.
+
+The runner is intentionally thin. It:
+
+1. Loads cases from disk (or builds them on demand by re-seeding).
+2. Sends each ``(question, pdf)`` pair to a vision-capable model.
+3. Scores the model's free-text answer against code-based ground truth.
+
+It does NOT do its own scoring methodology — that lives in
+:mod:`pdfhell.scorer`. It does NOT do its own provider dispatch — that
+lives in :mod:`multivon_eval.judge` (we reuse the vision-call dispatch
+from the multimodal evaluators).
+"""
+from __future__ import annotations
+
+import json
+from concurrent.futures import ThreadPoolExecutor, as_completed
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Iterable
+
+from multivon_eval import JudgeConfig
+
+from .case import HellCase
+from .scorer import CaseScore, SuiteReport, score_case, summarise
+from .vision import call_vision
+
+
+def parse_model_spec(spec: str) -> JudgeConfig:
+    """Parse ``"provider:model"`` shorthand into a ``JudgeConfig``.
+
+    Examples::
+
+        anthropic:claude-sonnet-4-6
+        openai:gpt-4o
+        google:gemini-2.5-pro
+
+    The shorthand is opinionated about temperature (0.0 — we want
+    deterministic answers for a benchmark) and max_tokens (256 — answers
+    should be short; long answers usually mean the model is rambling
+    around the answer rather than giving it).
+    """
+    if ":" not in spec:
+        raise ValueError(
+            f"model spec must be 'provider:model', got {spec!r}. "
+            "Example: anthropic:claude-sonnet-4-6"
+        )
+    provider, model = spec.split(":", 1)
+    provider = provider.strip().lower()
+    if provider not in {"anthropic", "openai", "google"}:
+        raise ValueError(
+            f"unsupported provider {provider!r}; "
+            "use anthropic, openai, or google"
+        )
+    # max_tokens=2048 gives prose answers room (e.g. footnote_override
+    # carve-out summaries) without letting models ramble. Gemini 2.5
+    # Flash allocates output tokens to internal "thinking"; tight
+    # budgets either produce empty responses or truncate mid-sentence.
+    # 2k is sufficient headroom in practice.
+    return JudgeConfig(
+        provider=provider,
+        model=model.strip(),
+        temperature=0.0,
+        max_tokens=2048,
+    )
+
+
+@dataclass(slots=True)
+class _Job:
+    case: HellCase
+    pdf_path: Path
+
+
+def _ask_model(job: _Job, judge: JudgeConfig) -> tuple[HellCase, str]:
+    """Send one case to the model. Returns (case, raw_answer).
+
+    Provider-level errors propagate as JudgeUnavailable. The CLI catches
+    them and records the case as refused.
+    """
+    answer = call_vision(
+        prompt=job.case.question,
+        sources=[str(job.pdf_path)],
+        judge=judge,
+        max_tokens=judge.max_tokens or 2048,
+    )
+    return job.case, answer.strip()
+
+
+def run_suite(
+    cases_dir: Path,
+    model_spec: str,
+    *,
+    workers: int = 4,
+    progress: bool = True,
+    suite_name: str = "mini",
+) -> SuiteReport:
+    """Evaluate every case under ``cases_dir`` against ``model_spec``.
+
+    ``cases_dir`` must contain ``<case_id>.json`` and ``<case_id>.pdf``
+    pairs produced by :func:`pdfhell.suite.build_suite`.
+    """
+    judge = parse_model_spec(model_spec)
+    jobs = list(_load_jobs(cases_dir))
+    if not jobs:
+        raise FileNotFoundError(f"no cases found in {cases_dir}")
+    scores: list[CaseScore] = []
+    with ThreadPoolExecutor(max_workers=workers) as pool:
+        futures = {pool.submit(_ask_model, job, judge): job for job in jobs}
+        completed = 0
+        for fut in as_completed(futures):
+            job = futures[fut]
+            try:
+                case, answer = fut.result()
+            except Exception as exc:  # JudgeUnavailable, provider error, etc.
+                # Treat upstream errors as refusals so the run still
+                # produces a complete report; the case is scored as
+                # incorrect with a refusal flag.
+                case = job.case
+                answer = f"[error] {type(exc).__name__}: {exc}"
+            score = score_case(case, answer)
+            scores.append(score)
+            completed += 1
+            if progress:
+                mark = "✓" if score.correct else ("⚠" if score.fell_for_trap else "✗")
+                print(f"  {mark} {score.case_id:36s}  expected={score.expected!r:30s}  got={answer[:60]!r}")
+    return summarise(model_spec, suite_name, scores)
+
+
+def _load_jobs(cases_dir: Path) -> Iterable[_Job]:
+    """Yield (case, pdf_path) pairs sorted by case id."""
+    for json_path in sorted(cases_dir.glob("*.json")):
+        case = HellCase.load_json(json_path)
+        pdf_path = json_path.with_suffix(".pdf")
+        if not pdf_path.exists():
+            # The case JSON tracks its own pdf_path relative to the
+            # suite root; honour that as a fallback.
+            pdf_path = (cases_dir / case.pdf_path).resolve()
+        if not pdf_path.exists():
+            raise FileNotFoundError(
+                f"PDF not found for case {case.id}; expected {pdf_path}"
+            )
+        yield _Job(case=case, pdf_path=pdf_path)

pdfhell/scorer.py

@@ -0,0 +1,214 @@
+"""Code-based scoring for pdfhell cases.
+
+LLM-as-judge is circular: the same complexity that fools the agent
+often fools the judge. pdfhell's primary correctness signal therefore
+does *not* go through an LLM. The PDF was generated from code, so the
+answer is exactly known and the scorer compares strings directly.
+
+QAG (multivon-eval's :class:`~multivon_eval.DocumentGrounding`) is
+available separately as the *explanation* of why a model failed — "the
+model returned $19,900.25, matching the hidden-OCR layer rather than
+the visible $18,900.25" — but it never affects pass/fail.
+"""
+from __future__ import annotations
+
+import re
+from dataclasses import dataclass, field
+from typing import Any
+
+from .case import HellCase
+
+
+_WHITESPACE_RE = re.compile(r"\s+")
+_PUNCT_NORMALIZE_RE = re.compile(r"[.,;:]+\s*$")
+
+
+def _normalize(s: str) -> str:
+    """Loose normalisation for free-text matching: collapse whitespace,
+    strip trailing punctuation, lower-case. Money strings stay
+    case-irrelevant; trailing periods get stripped so "The answer is
+    $18,900.25." matches "$18,900.25"."""
+    s = _WHITESPACE_RE.sub(" ", s.strip().lower())
+    s = _PUNCT_NORMALIZE_RE.sub("", s)
+    return s
+
+
+def _contains_loose(haystack: str, needle: str) -> bool:
+    return _normalize(needle) in _normalize(haystack)
+
+
+@dataclass(slots=True)
+class CaseScore:
+    """Score for one (case, model_output) pair.
+
+    ``correct`` is the headline binary signal pdfhell publishes. The
+    other fields disambiguate *how* the model got it wrong, which is
+    what makes pdfhell useful as a diagnostic tool — not just a number.
+    """
+
+    case_id: str
+    trap_family: str
+    correct: bool
+    fell_for_trap: bool
+    refused: bool
+    matched_expected: bool
+    matched_forbidden: list[str] = field(default_factory=list)
+    model_output: str = ""
+    expected: str = ""
+    failure_mode: str = ""  # human-readable, drawn from the case metadata when relevant
+
+    def to_dict(self) -> dict[str, Any]:
+        return {
+            "case_id": self.case_id,
+            "trap_family": self.trap_family,
+            "correct": self.correct,
+            "fell_for_trap": self.fell_for_trap,
+            "refused": self.refused,
+            "matched_expected": self.matched_expected,
+            "matched_forbidden": self.matched_forbidden,
+            "model_output": self.model_output,
+            "expected": self.expected,
+            "failure_mode": self.failure_mode,
+        }
+
+
+_REFUSAL_PATTERNS = (
+    "i cannot",
+    "i can't",
+    "unable to",
+    "i do not have access",
+    "i don't have access",
+    "no information",
+    "not visible",
+    "cannot determine",
+    "can't determine",
+)
+
+
+def _looks_like_refusal(output: str) -> bool:
+    low = output.strip().lower()
+    if len(low) < 4:
+        return True
+    return any(p in low for p in _REFUSAL_PATTERNS)
+
+
+def score_case(case: HellCase, model_output: str) -> CaseScore:
+    """Score a single (case, model_output) pair using code-based ground truth.
+
+    Decision tree:
+
+    - If the case has :attr:`HellCase.expected_tokens`, the output is
+      correct iff *every* token appears (whitespace-/case-tolerant).
+      Used for prose-style traps where the right answer can be phrased
+      multiple equally-valid ways but the *facts* are fixed (clause
+      numbers, dollar amounts, region codes).
+    - Otherwise, the output is correct iff it contains
+      :attr:`HellCase.expected_answer`. Used for single-value traps
+      (dollar amounts, dates, citations).
+    - Forbidden-answer detection runs regardless — if the model produces
+      a known wrong value, we record the diagnostic. A correct answer
+      that *also* contains a forbidden string is still wrong (it means
+      the model returned both, which is incoherent and should be flagged).
+    - Refusal detection runs last, only on otherwise-wrong outputs.
+    """
+    if case.expected_tokens:
+        matched_expected = all(_contains_loose(model_output, t) for t in case.expected_tokens)
+    else:
+        matched_expected = _contains_loose(model_output, case.expected_answer)
+    # When a case uses ``expected_tokens`` (prose answers), the
+    # ``forbidden_answer`` is often a literal substring of any complete
+    # correct answer (e.g. the body-only phrase is contained inside a
+    # full body+footnote summary). If all tokens matched, the model
+    # demonstrably captured the right facts; ignore the substring
+    # signal. For single-value traps (hidden OCR amount, table cell),
+    # expected and forbidden are mutually exclusive by construction, so
+    # the check stays active.
+    if case.expected_tokens and matched_expected:
+        matched_forbidden: list[str] = []
+    else:
+        matched_forbidden = [
+            f for f in case.forbidden_answers if _contains_loose(model_output, f)
+        ]
+    correct = matched_expected and not matched_forbidden
+    refused = (not matched_expected) and (not matched_forbidden) and _looks_like_refusal(model_output)
+    fell_for_trap = bool(matched_forbidden) and not matched_expected
+    failure_mode = ""
+    if fell_for_trap:
+        failure_mode = case.metadata.get("expected_failure_mode", "")
+    return CaseScore(
+        case_id=case.id,
+        trap_family=case.trap_family,
+        correct=correct,
+        fell_for_trap=fell_for_trap,
+        refused=refused,
+        matched_expected=matched_expected,
+        matched_forbidden=matched_forbidden,
+        model_output=model_output,
+        expected=case.expected_answer,
+        failure_mode=failure_mode,
+    )
+
+
+@dataclass(slots=True)
+class SuiteReport:
+    """Aggregate scoring summary for a run.
+
+    Designed to serialise to the leaderboard JSON multivon.ai already
+    consumes (one entry per (suite, model, judge) tuple).
+    """
+
+    model: str
+    suite: str
+    n: int
+    pass_rate: float
+    per_trap_pass: dict[str, float]
+    per_trap_fell_for_trap: dict[str, float]
+    refused_rate: float
+    cases: list[CaseScore] = field(default_factory=list)
+
+    def to_dict(self) -> dict[str, Any]:
+        return {
+            "model": self.model,
+            "suite": self.suite,
+            "n": self.n,
+            "pass_rate": self.pass_rate,
+            "per_trap_pass": self.per_trap_pass,
+            "per_trap_fell_for_trap": self.per_trap_fell_for_trap,
+            "refused_rate": self.refused_rate,
+            "cases": [c.to_dict() for c in self.cases],
+        }
+
+    def worst_failures(self, k: int = 5) -> list[CaseScore]:
+        """Return up to ``k`` cases that fell into the designed trap.
+
+        Useful for the launch share-card: "Worst failures: hidden-OCR
+        mismatch caught the model 8/10 times."
+        """
+        return [c for c in self.cases if c.fell_for_trap][:k]
+
+
+def summarise(model: str, suite: str, scores: list[CaseScore]) -> SuiteReport:
+    if not scores:
+        return SuiteReport(model=model, suite=suite, n=0, pass_rate=0.0,
+                           per_trap_pass={}, per_trap_fell_for_trap={}, refused_rate=0.0, cases=[])
+    by_trap: dict[str, list[CaseScore]] = {}
+    for s in scores:
+        by_trap.setdefault(s.trap_family, []).append(s)
+    per_trap_pass = {
+        trap: sum(c.correct for c in cs) / len(cs)
+        for trap, cs in by_trap.items()
+    }
+    per_trap_fell = {
+        trap: sum(c.fell_for_trap for c in cs) / len(cs)
+        for trap, cs in by_trap.items()
+    }
+    return SuiteReport(
+        model=model,
+        suite=suite,
+        n=len(scores),
+        pass_rate=sum(c.correct for c in scores) / len(scores),
+        per_trap_pass=per_trap_pass,
+        per_trap_fell_for_trap=per_trap_fell,
+        refused_rate=sum(c.refused for c in scores) / len(scores),
+        cases=scores,
+    )

pdfhell/suite.py

@@ -0,0 +1,104 @@
+"""Suite builder.
+
+Suites are sets of (PDF, case-JSON) pairs on disk. A suite is fully
+described by ``SuiteSpec`` — a deterministic recipe — so anyone with
+the suite name and ``pdfhell`` can re-derive byte-identical PDFs and
+answer keys.
+
+This is part of the "code-based ground truth" promise: the suite isn't
+a static blob, it's a recipe + a verifiable hash.
+"""
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from pathlib import Path
+from typing import Iterable
+
+from .case import HellCase
+from .generators import generate_case
+
+
+@dataclass(slots=True)
+class SuiteSpec:
+    """Recipe for a reproducible suite.
+
+    ``traps`` maps a trap family name to a list of seeds — those exact
+    seeds produce those exact PDFs. Run ``pdfhell build-suite --suite
+    mini`` to materialise to disk.
+    """
+
+    name: str
+    traps: dict[str, list[int]] = field(default_factory=dict)
+
+    @property
+    def total_cases(self) -> int:
+        return sum(len(s) for s in self.traps.values())
+
+
+def mini_suite() -> SuiteSpec:
+    """The canonical ``mini`` suite: 30 cases, 10 per trap family.
+
+    Seeds are arbitrary but fixed. The published leaderboard at
+    ``multivon.ai/leaderboard`` runs this exact spec — re-running it on
+    any machine produces identical PDFs.
+    """
+    return SuiteSpec(
+        name="mini",
+        traps={
+            "hidden_ocr_mismatch":      list(range(1001, 1011)),
+            "footnote_override":        list(range(2001, 2011)),
+            "split_table_across_pages": list(range(3001, 3011)),
+        },
+    )
+
+
+def smoke_suite() -> SuiteSpec:
+    """3-case quick-run for first-time users — one case per trap family.
+
+    Useful for ``uvx pdfhell run --suite smoke`` — runs in ~10 seconds
+    on Gemini Flash, costs fractions of a cent, and exercises every
+    trap family so a new user can see all three failure modes without
+    waiting for the full 30-case mini suite. Same seeds as the first
+    case in each mini-suite family, so smoke results are a strict
+    subset of mini results.
+    """
+    return SuiteSpec(
+        name="smoke",
+        traps={
+            "hidden_ocr_mismatch":      [1001],
+            "footnote_override":        [2001],
+            "split_table_across_pages": [3001],
+        },
+    )
+
+
+SUITES: dict[str, SuiteSpec] = {
+    "smoke": smoke_suite(),
+    "mini": mini_suite(),
+}
+
+
+def build_suite(spec: SuiteSpec, out_dir: Path) -> list[HellCase]:
+    """Materialise a suite to ``out_dir``.
+
+    Writes ``<case_id>.pdf`` and ``<case_id>.json`` pairs. Returns the
+    list of generated cases (with ``pdf_path`` set so callers can
+    serialise an index).
+    """
+    out_dir.mkdir(parents=True, exist_ok=True)
+    cases: list[HellCase] = []
+    for trap_family, seeds in spec.traps.items():
+        for seed in seeds:
+            pdf_bytes, case = generate_case(trap_family, seed)
+            pdf_path = out_dir / f"{case.id}.pdf"
+            pdf_path.write_bytes(pdf_bytes)
+            case.pdf_path = pdf_path.name  # relative — runners join with cases_dir
+            case.dump_json(out_dir / f"{case.id}.json")
+            cases.append(case)
+    return cases
+
+
+def iter_cases(cases_dir: Path) -> Iterable[HellCase]:
+    """Read every case from a materialised suite directory."""
+    for json_path in sorted(cases_dir.glob("*.json")):
+        yield HellCase.load_json(json_path)