pdfhell 0.1.0__py3-none-any.whl

pdfhell/vision.py

@@ -0,0 +1,231 @@
+"""Vision-call dispatch for the three supported providers.
+
+Each provider's vision API has a slightly different content-block
+shape (Anthropic wants ``image`` + ``document`` blocks; OpenAI wants
+``image_url`` + ``file``; Google wants inline ``Part.from_bytes``). We
+dispatch on :attr:`JudgeConfig.provider` and let the per-provider
+helpers do the format conversion. Errors surface as
+:class:`JudgeUnavailable` so the runner can treat upstream failures as
+refusals and still produce a complete report.
+"""
+from __future__ import annotations
+
+import base64
+import mimetypes
+import pathlib
+import re
+from typing import Iterable
+
+from multivon_eval import JudgeConfig
+from multivon_eval.exceptions import JudgeUnavailable
+
+
+# Models we know support vision input. Conservative: when in doubt we
+# don't gate (rely on the provider API to surface a real error).
+# Prefix-match — "gpt-5" catches gpt-5, gpt-5-mini, gpt-5.1, gpt-5.4, etc.
+_VISION_CAPABLE = {
+    "anthropic": {
+        "claude-haiku-4-5", "claude-sonnet-4-6", "claude-opus-4-7",
+        "claude-3-5-sonnet", "claude-3-5-haiku", "claude-3-opus",
+    },
+    "openai": {
+        "gpt-4o", "gpt-4.1", "gpt-5",
+    },
+    "google": {
+        "gemini-1.5", "gemini-2.5", "gemini-3", "gemini-3.1",
+    },
+}
+
+
+def _is_vision_capable(judge: JudgeConfig) -> bool:
+    model = (judge.model or "").lower()
+    if not model:
+        return True
+    for known in _VISION_CAPABLE.get(judge.provider, set()):
+        if model.startswith(known.lower()):
+            return True
+    return judge.provider not in _VISION_CAPABLE
+
+
+def _image_to_data_uri(src: str) -> tuple[str, str, str]:
+    """Return ``(uri_or_url, mime_type, base64_data)`` for an image source.
+
+    ``src`` may be ``http(s)://``, ``data:``, or a local filesystem path.
+    Local paths are read and inlined as base64.
+    """
+    if src.startswith("data:"):
+        match = re.match(r"data:([^;]+);base64,(.+)$", src)
+        if not match:
+            raise ValueError(f"unrecognised data URI: {src[:60]}")
+        return src, match.group(1), match.group(2)
+    if src.startswith("http://") or src.startswith("https://"):
+        mime = mimetypes.guess_type(src)[0] or "image/jpeg"
+        return src, mime, ""
+    path = pathlib.Path(src).expanduser().resolve()
+    if not path.is_file():
+        raise FileNotFoundError(f"image not found: {src}")
+    # PDFs are valid input to most vision APIs (they accept PDF mime
+    # types via the same image content blocks). We honour the actual
+    # extension rather than forcing image/jpeg.
+    suffix = path.suffix.lower()
+    if suffix == ".pdf":
+        mime = "application/pdf"
+    else:
+        mime = mimetypes.guess_type(path.name)[0] or "image/jpeg"
+    data = base64.b64encode(path.read_bytes()).decode("ascii")
+    return f"data:{mime};base64,{data}", mime, data
+
+
+def call_vision(
+    prompt: str,
+    sources: list[str],
+    judge: JudgeConfig,
+    max_tokens: int = 2048,
+) -> str:
+    """Call a vision-capable judge with a text prompt + one or more image
+    sources (paths, URLs, or data URIs). Returns the raw text answer.
+
+    Raises :class:`JudgeUnavailable` if the SDK is missing, an API key
+    isn't set, or the model is text-only.
+    """
+    if not _is_vision_capable(judge):
+        raise JudgeUnavailable(
+            f"vision-capable judge required; {judge.provider}/{judge.model} "
+            "is text-only. Try google:gemini-2.5-flash (cheap), "
+            "anthropic:claude-haiku-4-5, or openai:gpt-4o-mini."
+        )
+    provider = judge.provider
+    if provider == "anthropic":
+        return _anthropic_call(prompt, sources, judge, max_tokens)
+    if provider == "openai":
+        return _openai_call(prompt, sources, judge, max_tokens)
+    if provider == "google":
+        return _google_call(prompt, sources, judge, max_tokens)
+    raise JudgeUnavailable(
+        f"provider {provider!r} is not wired for vision; use "
+        "anthropic, openai, or google."
+    )
+
+
+def _anthropic_call(
+    prompt: str, sources: list[str], judge: JudgeConfig, max_tokens: int
+) -> str:
+    try:
+        import anthropic  # type: ignore[import-not-found]
+    except ImportError as exc:
+        raise JudgeUnavailable(
+            "anthropic SDK not installed. Install with `pip install 'pdfhell[anthropic]'` "
+            "or `pip install anthropic`."
+        ) from exc
+    content: list[dict] = []
+    for src in sources:
+        _, mime, b64 = _image_to_data_uri(src)
+        # PDF inputs use document content blocks on Anthropic; image
+        # inputs use image content blocks. Both encode as base64.
+        if mime == "application/pdf":
+            content.append({
+                "type": "document",
+                "source": {"type": "base64", "media_type": mime, "data": b64},
+            })
+        elif b64:
+            content.append({
+                "type": "image",
+                "source": {"type": "base64", "media_type": mime, "data": b64},
+            })
+        else:
+            content.append({"type": "image", "source": {"type": "url", "url": src}})
+    content.append({"type": "text", "text": prompt})
+    client = anthropic.Anthropic()
+    msg = client.messages.create(
+        model=judge.model,
+        max_tokens=max_tokens,
+        temperature=judge.temperature,
+        messages=[{"role": "user", "content": content}],
+    )
+    return "".join(b.text for b in msg.content if getattr(b, "type", "") == "text")
+
+
+def _openai_call(
+    prompt: str, sources: list[str], judge: JudgeConfig, max_tokens: int
+) -> str:
+    try:
+        import openai  # type: ignore[import-not-found]
+    except ImportError as exc:
+        raise JudgeUnavailable(
+            "openai SDK not installed. Install with `pip install 'pdfhell[openai]'` "
+            "or `pip install openai`."
+        ) from exc
+    parts: list[dict] = [{"type": "text", "text": prompt}]
+    for src in sources:
+        data_uri, mime, _ = _image_to_data_uri(src)
+        if mime == "application/pdf":
+            # OpenAI accepts PDFs via the file input type since GPT-4o
+            # (April 2025). The API expects an inline base64 data URI in
+            # the file part.
+            parts.append({"type": "file", "file": {"filename": pathlib.Path(src).name, "file_data": data_uri}})
+        else:
+            parts.append({"type": "image_url", "image_url": {"url": data_uri}})
+    client = openai.OpenAI(
+        base_url=judge.base_url if judge.base_url else None,
+    )
+
+    # GPT-5.x and the o-series reasoning models deprecated `max_tokens`
+    # in favour of `max_completion_tokens`, and they reject the legacy
+    # name with a 400. They also reserve some of the output budget for
+    # internal "thinking" tokens, so we double the cap for reasoning
+    # models to leave room for both reasoning and answer.
+    model = (judge.model or "").lower()
+    is_reasoning_model = model.startswith("gpt-5") or model.startswith("o1") or model.startswith("o3") or model.startswith("o4")
+    kwargs: dict = {
+        "model": judge.model,
+        "messages": [{"role": "user", "content": parts}],
+    }
+    if is_reasoning_model:
+        kwargs["max_completion_tokens"] = max_tokens * 2
+        # Reasoning models reject temperature != 1 with a 400. Omit the
+        # param entirely; the model picks its own default.
+    else:
+        kwargs["max_tokens"] = max_tokens
+        kwargs["temperature"] = judge.temperature
+
+    resp = client.chat.completions.create(**kwargs)
+    return resp.choices[0].message.content or ""
+
+
+def _google_call(
+    prompt: str, sources: list[str], judge: JudgeConfig, max_tokens: int
+) -> str:
+    try:
+        from google import genai  # type: ignore[import-not-found]
+        from google.genai import types as genai_types  # type: ignore[import-not-found]
+    except ImportError as exc:
+        raise JudgeUnavailable(
+            "google-genai SDK not installed. Install with `pip install 'pdfhell[google]'` "
+            "or `pip install google-genai`."
+        ) from exc
+    contents: list = []
+    for src in sources:
+        _, mime, b64 = _image_to_data_uri(src)
+        if b64:
+            contents.append(
+                genai_types.Part.from_bytes(data=base64.b64decode(b64), mime_type=mime)
+            )
+        else:
+            raise JudgeUnavailable(
+                "google-genai requires local files or data URIs for image input; "
+                f"got remote URL: {src}"
+            )
+    contents.append(prompt)
+    client = genai.Client()
+    resp = client.models.generate_content(
+        model=judge.model,
+        contents=contents,
+        config=genai_types.GenerateContentConfig(
+            temperature=judge.temperature,
+            max_output_tokens=max_tokens,
+        ),
+    )
+    return resp.text or ""
+
+
+__all__ = ["call_vision", "JudgeUnavailable"]

pdfhell-0.1.0.dist-info/METADATA

@@ -0,0 +1,208 @@
+Metadata-Version: 2.4
+Name: pdfhell
+Version: 0.1.0
+Summary: PDF Hell — adversarial PDFs that break AI document readers. Procedural ground truth, not LLM-as-judge.
+Author: Multivon
+License: Apache-2.0
+Project-URL: Homepage, https://pdfhell.multivon.ai
+Project-URL: Repository, https://github.com/multivon-ai/pdfhell
+Project-URL: Issues, https://github.com/multivon-ai/pdfhell/issues
+Project-URL: Leaderboard, https://pdfhell.multivon.ai/leaderboard
+Keywords: llm,evaluation,pdf,multimodal,benchmark,adversarial,document-ai,rag
+Classifier: License :: OSI Approved :: Apache Software License
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Operating System :: OS Independent
+Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
+Classifier: Topic :: Software Development :: Testing
+Requires-Python: >=3.10
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: multivon-eval>=0.7.2
+Requires-Dist: google-genai>=1.0
+Requires-Dist: reportlab>=4.0
+Requires-Dist: pypdf>=5.0
+Provides-Extra: all
+Dynamic: license-file
+
+# PDF Hell
+
+**Adversarial PDFs that break AI document readers — with procedural ground truth, not LLM-as-judge.**
+
+PDF Hell is a small, sharp benchmark for the "AI reads PDFs" claim. Every test case is a PDF generated *from code*, so the correct answer is known exactly. There's no LLM judging another LLM's interpretation — the same loop that fooled the model isn't asked to grade it.
+
+If your AI claims it can read documents, it should survive PDFs designed to break it.
+
+## Quickstart (30 seconds)
+
+```bash
+# 3-case smoke run against the cheapest vision model — works in any env with a Gemini key
+export GOOGLE_API_KEY=...
+uvx pdfhell run --model google:gemini-2.5-flash --suite smoke
+
+# Or run the full mini suite (30 cases, ~10s on Flash, ~$0.01)
+uvx pdfhell run --model anthropic:claude-sonnet-4-6 --suite mini
+
+# Or just generate one trap PDF and open it
+uvx pdfhell make --trap hidden_ocr_mismatch --seed 42
+open ./cases/hidden_ocr_mismatch-0042.pdf
+```
+
+That's it. `pdfhell run` builds the suite on first use, sends each PDF to the vision model, and grades the answer against code-based ground truth.
+
+Smoke result on Gemini 2.5 Flash (one case per family, run this minute):
+
+```
+PDF Hell smoke suite — n=3
+model: google:gemini-2.5-flash
+pass: 3/3  (100.0%)
+```
+
+## What's in the mini suite
+
+| Trap family | Cases | What breaks |
+|---|---|---|
+| `hidden_ocr_mismatch` | 10 | Invoices where the visible amount differs from an invisible OCR text layer. Vision-only models read the page; text-extraction pipelines read the layer; they disagree. |
+| `footnote_override` | 10 | Legal clauses where a 6pt footnote overrides the body — liability caps with carve-outs, terminations with restrictions, data-residency with disaster-recovery exceptions. |
+| `split_table_across_pages` | 10 | Financial tables where the header row sits on page 1 and the body rows on page 2. RAG loaders that paginate independently lose column context. |
+
+Every case has a deterministic seed. Re-running with the same seed regenerates **byte-identical PDFs** and identical answer keys. `Canvas(invariant=True)` is set on every generator so timestamps and document IDs don't drift between runs.
+
+The full suite (10 trap families, ~50 cases) is on the [roadmap](#roadmap).
+
+## Why this exists
+
+The current AI-eval state of the art uses an LLM-as-judge to grade another LLM's answer. That's circular: the same complexity that fools the agent fools the judge. PDF Hell rejects that:
+
+1. **Code-based ground truth.** The answer is a literal Python value the generator chose, not a frontier model's opinion.
+2. **A named failure mode per trap.** When a model fails, we know *which* specific failure caught it (e.g. "trusted the hidden OCR layer over the visible page").
+3. **A diagnostic signal**, not just a score. Per-trap-family breakdown tells you which assumption broke.
+
+## Commands
+
+```
+pdfhell list-traps                              # list trap families
+pdfhell make --trap <family> --seed <n>         # generate one case
+pdfhell build --suite <smoke|mini> --out <dir>  # materialise a suite
+pdfhell run --model <provider>:<model>          # evaluate a model
+  [--suite smoke|mini]                          #   (default: mini)
+  [--cases-dir <dir>]                           #   (default: ./cases/<suite>)
+  [--out <path>]                                #   JSON output
+  [--junit <path>]                              #   JUnit XML for GitHub Actions / GitLab CI
+  [--fail-threshold <0.0-1.0>]                  #   non-zero exit if pass_rate below threshold
+  [--workers <n>]                               #   parallel API requests (default: 4)
+  [--quiet]
+pdfhell report runs/<file>.json                 # print a saved run's summary
+```
+
+Provider shorthand: `anthropic:claude-sonnet-4-6`, `openai:gpt-4o`, `google:gemini-2.5-pro`, `google:gemini-2.5-flash`, etc. API key from env (`ANTHROPIC_API_KEY`, `OPENAI_API_KEY`, `GOOGLE_API_KEY`).
+
+## CI integration
+
+Drop this into `.github/workflows/eval.yml`:
+
+```yaml
+name: PDF Hell
+on: [pull_request]
+jobs:
+  pdfhell:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: astral-sh/setup-uv@v5
+      - run: uvx pdfhell run --model anthropic:claude-sonnet-4-6 --suite mini --junit results.xml --fail-threshold 0.7
+        env:
+          ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
+      - uses: actions/upload-artifact@v4
+        with:
+          name: pdfhell-results
+          path: results.xml
+```
+
+JUnit XML renders natively in the GitHub Actions / GitLab CI / CircleCI / Jenkins PR panel — failures show up as red rows with the expected and observed answers in the failure message.
+
+## How scoring works
+
+Two layers, applied in order:
+
+1. **Procedural exact match (primary)** — for single-value traps, the model's free-text answer must contain the expected value (whitespace-tolerant, case-insensitive). For prose traps like `footnote_override`, the model must include every required token (the cap value, every carve-out section number, etc.) in any order, in any phrasing. The model isn't graded on prose style; it's graded on whether it captured the facts.
+2. **Forbidden-answer detection (diagnostic)** — did the model return one of the answers the trap was specifically designed to elicit (e.g. the hidden-OCR amount)? If so, the trap caught a *known* failure mode and we record it. Doesn't affect the primary score.
+
+Anything that looks like a refusal (`"I can't determine..."`) is recorded as `refused`, not as a wrong answer.
+
+The QAG explanation layer from `multivon-eval` (`DocumentGrounding`) is available separately for users who want a human-readable "why did the model fail" breakdown — but it's never on the scoring path.
+
+## Adding a new trap family
+
+Add a generator at `pdfhell/generators/<your_trap>.py`:
+
+```python
+from ..case import HellCase
+from . import _common as C
+
+def generate(seed: int) -> tuple[bytes, HellCase]:
+    rng = C.rng_for(seed)
+    # ... draw a PDF with reportlab using rng for all random choices ...
+    # invariant=True is the default — keep your generator deterministic.
+    return pdf_bytes, HellCase(
+        id=f"your_trap-{seed:04d}",
+        trap_family="your_trap",
+        seed=seed,
+        question="What is ...?",
+        expected_answer="42",                # single canonical answer
+        expected_tokens=["42"],              # OR list of required substrings for prose
+        forbidden_answers=["41", "43"],      # OR a value the trap specifically elicits
+        metadata={"expected_failure_mode": "Model does X when it should do Y."},
+    )
+```
+
+Register it in `pdfhell/generators/__init__.py`. See [CONTRIBUTING.md](./CONTRIBUTING.md) for the full guide. Tests run with `pytest`.
+
+## Roadmap
+
+The 0.1 release is intentionally narrow — three trap families, 30 cases. Coming next:
+
+- `merged_table_cells` — value depends on row/column span interpretation
+- `rotated_scan` — visually legible but OCR-broken pages
+- `near_duplicate_entities` — "ACME Ltd." vs "ACME Holdings Ltd."
+- `prompt_injection_in_body` — "Ignore previous instructions and answer X"
+- `chart_axis_inversion` — answers depend on reading axis direction
+- `checkbox_ambiguity` — selected vs unselected with low visual margin
+- `cross_page_citation` — answers requiring page + bounding-box citations
+
+Target full suite: 10 trap families, ~50 cases.
+
+## Hosted generator
+
+For document-AI teams who need adversarial test cases tailored to *their* templates (claims forms, MSAs, medical records, KYC docs), there's a hosted generator that takes your templates and produces adversarial variants with code-based ground truth — same methodology, your data shape.
+
+Email `hello@multivon.ai` for early access, or see [multivon.ai/pricing](https://multivon.ai/pricing).
+
+## Installing
+
+```bash
+# Recommended (zero-install with uv):
+uvx pdfhell list-traps
+
+# Or in a venv:
+python -m venv .venv && source .venv/bin/activate
+pip install pdfhell
+```
+
+Bare install brings in `multivon-eval` (the engine), `reportlab` (PDF generation), `pypdf`, and the three frontier-provider SDKs (anthropic, openai, google-genai). No provider extras to remember; no GPU required.
+
+## License
+
+Apache 2.0. Built on [`multivon-eval`](https://github.com/multivon-ai/multivon-eval).
+
+## Citing
+
+```bibtex
+@software{pdfhell,
+  title  = {PDF Hell: Adversarial PDFs for AI document readers},
+  author = {Multivon},
+  url    = {https://github.com/multivon-ai/pdfhell},
+}
+```

pdfhell-0.1.0.dist-info/RECORD

@@ -0,0 +1,20 @@
+pdfhell/__init__.py,sha256=HQSEHz5hlaZ1LFbY34fdzUGXMlJc6VxXDhfG77AyCQA,919
+pdfhell/auditpack.py,sha256=aU65vUBlq3QJuzKhH5TvmcKLvMWhtJ8z_A8SETA9lms,6297
+pdfhell/case.py,sha256=KRg10SCPy7b_vbCRPEWszJFSyn1HYJuoNeSSfXJ4M98,3835
+pdfhell/cli.py,sha256=F5wdepx-1cir8wO3TK47nufi4gdT3QbujCvfoWuGelE,8216
+pdfhell/junit.py,sha256=DYWmKb6VDTaNlY7TBGfQS7Hu24qsO7hJULB6FXL9eEc,3580
+pdfhell/runner.py,sha256=Tr_Zdnewn8lkkNEV3MPRtqjZEj5exke7y1k641ctBu8,5119
+pdfhell/scorer.py,sha256=cMgnzks5fX-Hz9c-aYsx3ZwbpQfy7Gv0xMfElT2YD0U,7711
+pdfhell/suite.py,sha256=VQkEi4CEuE1vdxRzXj804MlWXM-infiB0xFs2YMeUrg,3382
+pdfhell/vision.py,sha256=HUEpNFVvaFAI1mTwBlp9zIh-94gYCknVrCCve-BamRg,8781
+pdfhell/generators/__init__.py,sha256=wmYbtoI_G6_-J7TaWyIvQ3887YJyDI2gGwNxZjWoGVE,1841
+pdfhell/generators/_common.py,sha256=OlSmOtymawWXUjP1UbcYEkSRKxezm7AltH6rnacOaSk,6112
+pdfhell/generators/footnote_override.py,sha256=9S703znrSZy2SDQ5-p5sMgpRmUDh3szQmTe6WaYnl-c,8835
+pdfhell/generators/hidden_ocr_mismatch.py,sha256=Q3Lx5SZRH_Zywa_vSNkUu9fvFWsI_m0wKXyIIxyaSTE,4992
+pdfhell/generators/split_table_across_pages.py,sha256=1bWL5kv8CNUR6bbgfTNkzXtfoMbRNImQk6dGdkSYaw4,6293
+pdfhell-0.1.0.dist-info/licenses/LICENSE,sha256=42Dg0T3y9vfd-um9B8j26KeAXJrE3jgtK_ggMrcHS9o,737
+pdfhell-0.1.0.dist-info/METADATA,sha256=TbFR-FnLd2Tbs5sw7vhVGieOLUFBBcf5Ci3Q8OlLU8U,9691
+pdfhell-0.1.0.dist-info/WHEEL,sha256=aeYiig01lYGDzBgS8HxWXOg3uV61G9ijOsup-k9o1sk,91
+pdfhell-0.1.0.dist-info/entry_points.txt,sha256=LiHsXd67OOxVUSAT9N-EQARvAXnoHlGhOW8R1QclsAs,45
+pdfhell-0.1.0.dist-info/top_level.txt,sha256=kLeB8Xx5QVEC_p8kAoAEg-wB80unVpYb50ELR7QO_Rg,8
+pdfhell-0.1.0.dist-info/RECORD,,

pdfhell-0.1.0.dist-info/WHEEL

@@ -0,0 +1,5 @@
+Wheel-Version: 1.0
+Generator: setuptools (82.0.1)
+Root-Is-Purelib: true
+Tag: py3-none-any
+

pdfhell-0.1.0.dist-info/entry_points.txt

@@ -0,0 +1,2 @@
+[console_scripts]
+pdfhell = pdfhell.cli:main

pdfhell-0.1.0.dist-info/licenses/LICENSE

@@ -0,0 +1,17 @@
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   Copyright 2026 Multivon
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.

pdfhell-0.1.0.dist-info/top_level.txt

@@ -0,0 +1 @@
+pdfhell