pdfhell 0.1.0__py3-none-any.whl

pdfhell/__init__.py

@@ -0,0 +1,34 @@
+"""PDF Hell — adversarial PDFs that break AI document readers.
+
+Procedural ground truth, not LLM-as-judge. Each trap family generates PDFs
+*from code*, so the answer key is exact and reproducible — no circular
+assurance.
+
+Quickstart::
+
+    uvx pdfhell make --trap hidden_ocr_mismatch --seed 42
+    uvx pdfhell run --model anthropic:claude-sonnet-4-6 --suite mini
+    uvx pdfhell report runs/claude.json --share-card
+
+Build on top of ``multivon-eval`` (the QAG engine, provider adapters, audit
+packaging, cost tracking). pdfhell is *only* the adversarial generation
+layer; the runtime, scoring, and reporting come from multivon-eval.
+"""
+from __future__ import annotations
+
+__version__ = "0.1.0"
+
+from .case import HellCase
+from .generators import (
+    GENERATORS,
+    TRAP_FAMILIES,
+    generate_case,
+)
+
+__all__ = [
+    "__version__",
+    "HellCase",
+    "GENERATORS",
+    "TRAP_FAMILIES",
+    "generate_case",
+]

pdfhell/auditpack.py

@@ -0,0 +1,182 @@
+"""Build a downloadable, hash-chained audit pack from a pdfhell run.
+
+The pack is a ZIP containing:
+
+- ``manifest.json`` — pdfhell version, run timestamp, model spec, suite,
+  per-trap pass rates, total cost (when known), SHA-256 of every file
+  inside the pack.
+- ``run.json`` — the full :class:`SuiteReport` JSON.
+- ``run.xml`` — JUnit XML (same data as ``run.json``, machine-readable
+  for CI dashboards).
+- ``cases/<case_id>.pdf`` — every adversarial PDF the model was tested
+  against.
+- ``cases/<case_id>.json`` — each case's answer key + metadata.
+- ``README.txt`` — human-readable "what's in this ZIP" + reproduction
+  command. Procurement teams open this first.
+
+The audit pack is the artifact a buyer's procurement team attaches to
+a diligence appendix. It must be self-describing (no out-of-band
+context required), reproducible (the manifest tells you the exact
+command to regenerate the run), and tamper-evident (the manifest
+includes a SHA-256 for every file in the pack; auditors can verify the
+ZIP wasn't edited after delivery).
+"""
+from __future__ import annotations
+
+import hashlib
+import json
+import zipfile
+from datetime import datetime, timezone
+from pathlib import Path
+from typing import Iterable
+
+from . import __version__
+from .case import HellCase
+from .junit import report_to_junit
+from .scorer import SuiteReport
+
+
+_README_TEMPLATE = """\
+# pdfhell audit pack
+
+This ZIP is a complete, self-describing record of one PDF Hell run. It
+contains every PDF the model was asked to read, every answer key, the
+raw model output, and a tamper-evident manifest.
+
+## What's in this pack
+
+- manifest.json — Run metadata + SHA-256 of every file in this ZIP.
+- run.json — Full run report (per-case scores, model outputs).
+- run.xml — JUnit XML (renders in CI dashboards).
+- cases/*.pdf — The adversarial PDFs the model was tested against.
+- cases/*.json — The answer keys + per-case metadata.
+- README.txt — This file.
+
+## How to verify
+
+The manifest contains a SHA-256 for every file in this ZIP. To verify
+nothing was edited after delivery:
+
+  unzip -p audit-pack.zip manifest.json | jq .files
+  sha256sum cases/*.pdf cases/*.json run.json run.xml README.txt
+
+Each hash in the manifest must match the file's actual SHA-256.
+
+## How to reproduce
+
+The manifest records the exact pdfhell command. To regenerate
+byte-identical PDFs and re-run the same model:
+
+  {repro_command}
+
+pdfhell uses Canvas(invariant=True) on every generator so PDFs are
+byte-identical across runs with the same seed.
+
+## Scope
+
+pdfhell {pdfhell_version}, suite {suite}, model {model}. Generated
+{timestamp}. {n} cases, {passed}/{n} passed ({pass_rate:.0%}). See
+manifest.json for per-trap breakdown.
+"""
+
+
+def _sha256(data: bytes) -> str:
+    return hashlib.sha256(data).hexdigest()
+
+
+def _gather_files(report: SuiteReport, cases_dir: Path) -> Iterable[tuple[str, bytes]]:
+    """Yield (arcname, bytes) pairs for every file going into the ZIP.
+
+    Order: README first (humans see it first), then manifest, then JSON
+    + XML, then case PDFs + answer keys. Stable ordering keeps the
+    SHA-256 of the ZIP itself stable across runs.
+    """
+    for case_summary in report.cases:
+        case_id = case_summary.case_id
+        pdf_path = cases_dir / f"{case_id}.pdf"
+        json_path = cases_dir / f"{case_id}.json"
+        if pdf_path.exists():
+            yield f"cases/{case_id}.pdf", pdf_path.read_bytes()
+        if json_path.exists():
+            yield f"cases/{case_id}.json", json_path.read_bytes()
+
+
+def build_audit_pack(
+    report: SuiteReport,
+    cases_dir: Path,
+    out_path: Path,
+) -> Path:
+    """Write a complete audit ZIP for ``report`` to ``out_path``.
+
+    Returns the resolved output path.
+    """
+    out_path = out_path.resolve()
+    out_path.parent.mkdir(parents=True, exist_ok=True)
+
+    # Materialise the per-case files into bytes first so we can hash them.
+    case_files: list[tuple[str, bytes]] = list(_gather_files(report, cases_dir))
+
+    run_json_bytes = json.dumps(report.to_dict(), indent=2).encode("utf-8")
+    run_xml_bytes = report_to_junit(report).encode("utf-8")
+    timestamp = datetime.now(timezone.utc).isoformat()
+    passed = sum(1 for c in report.cases if c.correct)
+
+    repro_command = (
+        f"uvx pdfhell run --model {report.model} --suite {report.suite}"
+    )
+    readme_bytes = _README_TEMPLATE.format(
+        pdfhell_version=__version__,
+        suite=report.suite,
+        model=report.model,
+        timestamp=timestamp,
+        n=report.n,
+        passed=passed,
+        pass_rate=report.pass_rate,
+        repro_command=repro_command,
+    ).encode("utf-8")
+
+    # Build a manifest that hashes every other file in the pack. The
+    # manifest is the LAST file we hash so we can include the hashes of
+    # everything else inside it.
+    files_in_pack: list[tuple[str, bytes]] = [
+        ("README.txt", readme_bytes),
+        ("run.json", run_json_bytes),
+        ("run.xml", run_xml_bytes),
+        *case_files,
+    ]
+    manifest = {
+        "pdfhell_version": __version__,
+        "generated_at": timestamp,
+        "model": report.model,
+        "suite": report.suite,
+        "n": report.n,
+        "passed": passed,
+        "pass_rate": report.pass_rate,
+        "per_trap_pass": report.per_trap_pass,
+        "per_trap_fell_for_trap": report.per_trap_fell_for_trap,
+        "reproduction": {
+            "command": repro_command,
+            "note": (
+                "PDFs are regenerated byte-identically via Canvas(invariant=True). "
+                "Same seed → same PDF → same answer key."
+            ),
+        },
+        "files": [
+            {"path": name, "sha256": _sha256(data), "size": len(data)}
+            for name, data in files_in_pack
+        ],
+    }
+    manifest_bytes = json.dumps(manifest, indent=2).encode("utf-8")
+
+    # ZIP_DEFLATED is universal; mtime is set to the run timestamp so
+    # the ZIP itself is reproducible across packaging runs.
+    with zipfile.ZipFile(out_path, "w", compression=zipfile.ZIP_DEFLATED) as zf:
+        for name, data in [("manifest.json", manifest_bytes), *files_in_pack]:
+            info = zipfile.ZipInfo(name)
+            info.date_time = (2026, 1, 1, 0, 0, 0)
+            zf.writestr(info, data)
+
+    return out_path
+
+
+__all__ = ["build_audit_pack"]

pdfhell/case.py

@@ -0,0 +1,87 @@
+"""Canonical case representation for a generated pdfhell trap.
+
+A ``HellCase`` is the serialisable artifact that every trap-family generator
+returns. The runner consumes these; the scoring layer treats
+``expected_answer`` as code-based ground truth and grades the model's
+free-text output against it. QAG (multivon-eval's
+:class:`~multivon_eval.DocumentGrounding`) is layered on as the
+*explanation* of why a particular answer scored a particular way — not as
+the score itself. This is the architectural fix the Round-7 churned design
+partner demanded: no LLM-judges-LLM circular assurance for the primary
+correctness signal.
+"""
+from __future__ import annotations
+
+import json
+from dataclasses import asdict, dataclass, field
+from pathlib import Path
+from typing import Any
+
+
+@dataclass(slots=True)
+class HellCase:
+    """One adversarial PDF + its ground truth.
+
+    Attributes
+    ----------
+    id: stable, deterministic, e.g. ``"hidden_ocr_mismatch-0042"``.
+    trap_family: one of :data:`pdfhell.generators.TRAP_FAMILIES`.
+    seed: integer seed used to generate the case. Regenerating with the
+        same seed produces a byte-identical PDF and identical answer key.
+    question: the user-facing question the model must answer.
+    expected_answer: a human-readable form of the correct answer used in
+        reports + JUnit output. For single-value traps this is also the
+        substring the scorer looks for; for prose-style traps (e.g.
+        :mod:`pdfhell.generators.footnote_override`) the scorer instead
+        uses :attr:`expected_tokens` (see below).
+    expected_tokens: optional list of substrings that ALL must appear in
+        the model's output for the case to count as correct. Used by
+        traps where the right answer can be expressed multiple
+        equally-valid ways (e.g. a list of clause carve-outs in any
+        order). When empty, the scorer falls back to a contains-match
+        against ``expected_answer``.
+    forbidden_answers: optional list of plausible-but-wrong answers the
+        trap aims to elicit (e.g. the hidden-OCR amount). Used by the
+        scorer to detect the specific failure mode the trap was designed
+        for.
+    pdf_path: location of the generated PDF (relative to the suite root).
+    metadata: extra fields — number of pages, font size of footnotes,
+        the literal hidden-OCR string, etc. Useful for diagnostics and
+        for trap-family-specific scorers.
+    """
+
+    id: str
+    trap_family: str
+    seed: int
+    question: str
+    expected_answer: str
+    forbidden_answers: list[str] = field(default_factory=list)
+    expected_tokens: list[str] = field(default_factory=list)
+    pdf_path: str = ""
+    metadata: dict[str, Any] = field(default_factory=dict)
+
+    # ─── serialisation ─────────────────────────────────────────────────────
+
+    def to_dict(self) -> dict[str, Any]:
+        return asdict(self)
+
+    @classmethod
+    def from_dict(cls, raw: dict[str, Any]) -> "HellCase":
+        return cls(
+            id=raw["id"],
+            trap_family=raw["trap_family"],
+            seed=int(raw["seed"]),
+            question=raw["question"],
+            expected_answer=raw["expected_answer"],
+            forbidden_answers=list(raw.get("forbidden_answers", [])),
+            expected_tokens=list(raw.get("expected_tokens", [])),
+            pdf_path=raw.get("pdf_path", ""),
+            metadata=dict(raw.get("metadata", {})),
+        )
+
+    def dump_json(self, path: str | Path) -> None:
+        Path(path).write_text(json.dumps(self.to_dict(), indent=2), encoding="utf-8")
+
+    @classmethod
+    def load_json(cls, path: str | Path) -> "HellCase":
+        return cls.from_dict(json.loads(Path(path).read_text(encoding="utf-8")))

pdfhell/cli.py

@@ -0,0 +1,216 @@
+"""pdfhell CLI.
+
+Four subcommands keep the surface minimal:
+
+    pdfhell list-traps                       list available trap families
+    pdfhell make --trap X --seed N [--out P] generate one case (pdf + json)
+    pdfhell build --suite mini --out cases/  materialise a named suite
+    pdfhell run --suite mini --model ...     evaluate a model against the suite
+    pdfhell report runs/<name>.json          print summary + optional share card
+
+Everything else (scoring, provider dispatch, audit packaging) is pulled
+from :mod:`multivon_eval`. The CLI is glue.
+"""
+from __future__ import annotations
+
+import argparse
+import json
+import sys
+from pathlib import Path
+
+from . import __version__
+from .auditpack import build_audit_pack
+from .case import HellCase
+from .generators import TRAP_FAMILIES, generate_case
+from .junit import report_to_junit
+from .runner import parse_model_spec, run_suite
+from .scorer import SuiteReport
+from .suite import SUITES, build_suite
+
+
+def _cmd_list_traps(args: argparse.Namespace) -> int:
+    for family in TRAP_FAMILIES:
+        print(family)
+    return 0
+
+
+def _cmd_make(args: argparse.Namespace) -> int:
+    try:
+        pdf_bytes, case = generate_case(args.trap, args.seed)
+    except KeyError as exc:
+        print(exc, file=sys.stderr)
+        print(f"available trap families: {', '.join(TRAP_FAMILIES)}", file=sys.stderr)
+        return 2
+    out_dir = Path(args.out).resolve()
+    out_dir.mkdir(parents=True, exist_ok=True)
+    pdf_path = out_dir / f"{case.id}.pdf"
+    json_path = out_dir / f"{case.id}.json"
+    pdf_path.write_bytes(pdf_bytes)
+    case.pdf_path = pdf_path.name
+    case.dump_json(json_path)
+    print(f"wrote {pdf_path}  ({len(pdf_bytes):,} bytes)")
+    print(f"wrote {json_path}")
+    print(f"expected answer: {case.expected_answer}")
+    return 0
+
+
+def _cmd_build(args: argparse.Namespace) -> int:
+    if args.suite not in SUITES:
+        print(f"unknown suite {args.suite!r}; available: {', '.join(SUITES)}", file=sys.stderr)
+        return 2
+    spec = SUITES[args.suite]
+    out_dir = Path(args.out).resolve()
+    print(f"building suite {spec.name!r} ({spec.total_cases} cases) → {out_dir}")
+    cases = build_suite(spec, out_dir)
+    print(f"wrote {len(cases)} cases")
+    return 0
+
+
+def _cmd_run(args: argparse.Namespace) -> int:
+    # Default cases dir tracks the suite name so `--suite smoke` doesn't
+    # silently run the mini suite.
+    cases_dir = Path(args.cases_dir).resolve() if args.cases_dir else Path(f"./cases/{args.suite}").resolve()
+    if not cases_dir.is_dir():
+        if args.suite in SUITES:
+            print(f"cases dir {cases_dir} not found; building {args.suite} suite first ...")
+            cases_dir.mkdir(parents=True, exist_ok=True)
+            build_suite(SUITES[args.suite], cases_dir)
+        else:
+            print(f"cases dir {cases_dir} not found and suite {args.suite!r} is unknown",
+                  file=sys.stderr)
+            return 2
+
+    print(f"running {args.model} against {args.suite} suite at {cases_dir}")
+    report = run_suite(
+        cases_dir=cases_dir,
+        model_spec=args.model,
+        workers=args.workers,
+        progress=not args.quiet,
+        suite_name=args.suite,
+    )
+    out_path = Path(args.out).resolve() if args.out else _default_run_path(args.model, args.suite)
+    out_path.parent.mkdir(parents=True, exist_ok=True)
+    out_path.write_text(json.dumps(report.to_dict(), indent=2), encoding="utf-8")
+    print()
+    _print_report(report)
+    print()
+    print(f"wrote {out_path}")
+    if args.junit:
+        junit_path = Path(args.junit).resolve()
+        junit_path.parent.mkdir(parents=True, exist_ok=True)
+        junit_path.write_text(report_to_junit(report), encoding="utf-8")
+        print(f"wrote {junit_path}")
+    if args.audit_pack:
+        zip_path = Path(args.audit_pack).resolve()
+        build_audit_pack(report, cases_dir, zip_path)
+        print(f"wrote {zip_path}")
+    if args.fail_threshold is not None and report.pass_rate < args.fail_threshold:
+        print(
+            f"\nFAIL: pass_rate {report.pass_rate:.1%} below --fail-threshold "
+            f"{args.fail_threshold:.1%}"
+        )
+        return 1
+    return 0
+
+
+def _cmd_report(args: argparse.Namespace) -> int:
+    raw = json.loads(Path(args.run).read_text(encoding="utf-8"))
+    report = SuiteReport(
+        model=raw["model"],
+        suite=raw["suite"],
+        n=raw["n"],
+        pass_rate=raw["pass_rate"],
+        per_trap_pass=raw["per_trap_pass"],
+        per_trap_fell_for_trap=raw["per_trap_fell_for_trap"],
+        refused_rate=raw["refused_rate"],
+        cases=[],  # not needed for printing the summary
+    )
+    _print_report(report)
+    return 0
+
+
+def _print_report(report: SuiteReport) -> None:
+    print(f"PDF Hell {report.suite} suite — n={report.n}")
+    print()
+    print(f"model: {report.model}")
+    print(f"pass: {sum(1 for _ in report.cases if _.correct) if report.cases else int(report.pass_rate * report.n)}/{report.n}  ({report.pass_rate:.1%})")
+    print(f"refused: {report.refused_rate:.1%}")
+    print()
+    print("per-trap pass rate:")
+    for trap, rate in sorted(report.per_trap_pass.items()):
+        fell = report.per_trap_fell_for_trap.get(trap, 0.0)
+        print(f"  {trap:30s}  pass={rate:.0%}  fell-for-trap={fell:.0%}")
+
+
+def _default_run_path(model_spec: str, suite: str) -> Path:
+    safe = model_spec.replace("/", "-").replace(":", "-")
+    return Path(f"runs/{suite}-{safe}.json").resolve()
+
+
+def build_parser() -> argparse.ArgumentParser:
+    p = argparse.ArgumentParser(
+        prog="pdfhell",
+        description="PDF Hell — adversarial PDFs that break AI document readers.",
+    )
+    p.add_argument("--version", action="version", version=f"pdfhell {__version__}")
+    sub = p.add_subparsers(dest="cmd", required=True, metavar="<command>")
+
+    p_list = sub.add_parser("list-traps", help="list available trap families")
+    p_list.set_defaults(func=_cmd_list_traps)
+
+    p_make = sub.add_parser("make", help="generate one case (pdf + json)")
+    p_make.add_argument("--trap", required=True, choices=TRAP_FAMILIES)
+    p_make.add_argument("--seed", required=True, type=int)
+    p_make.add_argument("--out", default="./cases", help="output directory (default: ./cases)")
+    p_make.set_defaults(func=_cmd_make)
+
+    p_build = sub.add_parser("build", help="materialise a named suite to disk")
+    p_build.add_argument("--suite", default="mini", choices=tuple(SUITES.keys()))
+    p_build.add_argument("--out", default="./cases/mini")
+    p_build.set_defaults(func=_cmd_build)
+
+    p_run = sub.add_parser("run", help="evaluate a model against a suite")
+    p_run.add_argument("--model", required=True,
+                       help="provider:model, e.g. anthropic:claude-sonnet-4-6")
+    p_run.add_argument("--suite", default="mini", choices=tuple(SUITES.keys()))
+    p_run.add_argument(
+        "--cases-dir",
+        default=None,
+        help="dir with materialised cases (default: ./cases/<suite>; built on demand if missing)",
+    )
+    p_run.add_argument("--workers", type=int, default=4)
+    p_run.add_argument("--quiet", action="store_true")
+    p_run.add_argument("--out", help="output JSON path (default: runs/<suite>-<model>.json)")
+    p_run.add_argument(
+        "--junit",
+        help="also write a JUnit XML report to this path (renders in GitHub Actions / GitLab CI)",
+    )
+    p_run.add_argument(
+        "--audit-pack",
+        help=(
+            "also write a complete, hash-chained audit ZIP to this path "
+            "(PDFs + answer keys + run JSON + JUnit XML + SHA-256 manifest)"
+        ),
+    )
+    p_run.add_argument(
+        "--fail-threshold",
+        type=float,
+        help="exit nonzero if pass_rate is below this fraction (0.0–1.0); for CI gates",
+    )
+    p_run.set_defaults(func=_cmd_run)
+
+    p_report = sub.add_parser("report", help="print summary from a run JSON")
+    p_report.add_argument("run", help="path to runs/<suite>-<model>.json")
+    p_report.set_defaults(func=_cmd_report)
+
+    return p
+
+
+def main(argv: list[str] | None = None) -> int:
+    parser = build_parser()
+    args = parser.parse_args(argv)
+    return args.func(args)
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

pdfhell/generators/__init__.py

@@ -0,0 +1,49 @@
+"""Trap-family generators.
+
+Each generator is a deterministic ``(seed) -> (pdf_bytes, HellCase)``
+function. The mapping :data:`GENERATORS` powers ``pdfhell make --trap X``
+and the suite builder. Adding a new trap family means registering it
+here.
+
+Why a registry and not subclasses? Each trap is fundamentally a *small*
+parameterised generator. The registry keeps generators portable, makes
+the CLI's ``--trap`` flag introspectable, and avoids the class-hierarchy
+sprawl that a 50-trap eventual suite would otherwise turn into.
+"""
+from __future__ import annotations
+
+from typing import Callable
+
+from ..case import HellCase
+from .hidden_ocr_mismatch import generate as _hidden_ocr_mismatch
+from .footnote_override import generate as _footnote_override
+from .split_table_across_pages import generate as _split_table_across_pages
+
+
+# Signature: (seed: int) -> (pdf_bytes: bytes, case: HellCase).
+# The case's pdf_path is set by the suite-builder after writing the bytes.
+GeneratorFn = Callable[[int], tuple[bytes, HellCase]]
+
+GENERATORS: dict[str, GeneratorFn] = {
+    "hidden_ocr_mismatch": _hidden_ocr_mismatch,
+    "footnote_override": _footnote_override,
+    "split_table_across_pages": _split_table_across_pages,
+}
+
+TRAP_FAMILIES: tuple[str, ...] = tuple(GENERATORS.keys())
+
+
+def generate_case(trap_family: str, seed: int) -> tuple[bytes, HellCase]:
+    """Generate one case from ``trap_family`` with the given ``seed``.
+
+    Raises :class:`KeyError` for an unknown trap family — the CLI catches
+    this and prints the list of available families.
+    """
+    if trap_family not in GENERATORS:
+        raise KeyError(
+            f"unknown trap family {trap_family!r}; available: {', '.join(TRAP_FAMILIES)}"
+        )
+    return GENERATORS[trap_family](seed)
+
+
+__all__ = ["GENERATORS", "TRAP_FAMILIES", "generate_case", "GeneratorFn"]