fieldbench 0.2.1__py3-none-any.whl

fieldbench/__init__.py

@@ -0,0 +1,23 @@
+"""FieldBench — a cross-domain, field-level benchmark for schema-driven document extraction."""
+
+from .aggregate import DocResult, Report, build_report
+from .corpus import list_documents, score_corpus
+from .run import LLMRunner, Runner, run_corpus
+from .scoring import SCORER_VERSION, FieldResult, compare_field
+
+__version__ = "0.2.1"
+
+__all__ = [
+    "compare_field",
+    "FieldResult",
+    "SCORER_VERSION",
+    "score_corpus",
+    "list_documents",
+    "build_report",
+    "DocResult",
+    "Report",
+    "run_corpus",
+    "Runner",
+    "LLMRunner",
+    "__version__",
+]

fieldbench/aggregate.py

@@ -0,0 +1,120 @@
+"""Aggregate per-field results into a FieldBench report.
+
+Reports micro/macro field accuracy, always stratified real vs synthetic, with a
+four-way confusion breakdown (correct-absence / hallucination / miss /
+wrong-value) and the all-null degenerate floor — so every score is read against
+what an empty extractor scores for free.
+"""
+
+from __future__ import annotations
+
+from collections import Counter, defaultdict
+from dataclasses import dataclass, field
+
+from .scoring import SCORER_VERSION, FieldResult, is_empty
+
+
+@dataclass
+class DocResult:
+    stem: str
+    category: str
+    source: str  # "real" | "synthetic" | "unknown"
+    fields: list[FieldResult]
+
+
+@dataclass
+class Stratum:
+    docs: int = 0
+    fields: int = 0
+    passed: int = 0
+    weighted: float = 0.0
+    four_way: Counter = field(default_factory=Counter)
+
+    @property
+    def accuracy(self) -> float:
+        return self.passed / self.fields if self.fields else 0.0
+
+    @property
+    def partial(self) -> float:
+        return self.weighted / self.fields if self.fields else 0.0
+
+    def to_dict(self) -> dict:
+        n = self.fields or 1
+        return {
+            "docs": self.docs,
+            "fields": self.fields,
+            "passed": self.passed,
+            "accuracy": round(self.accuracy, 4),
+            "partial_credit": round(self.partial, 4),
+            "four_way": {
+                bucket: {"count": c, "rate": round(c / n, 4)}
+                for bucket, c in sorted(self.four_way.items())
+            },
+        }
+
+
+@dataclass
+class Report:
+    scorer_version: str = SCORER_VERSION
+    mode: str = "unspecified"  # which representation the predictions came from
+    overall: Stratum = field(default_factory=Stratum)
+    by_source: dict[str, Stratum] = field(default_factory=dict)
+    by_category: dict[str, Stratum] = field(default_factory=dict)
+    four_way: Counter = field(default_factory=Counter)
+    all_null_floor: float = 0.0
+
+    def to_dict(self) -> dict:
+        total = sum(self.four_way.values()) or 1
+        return {
+            "scorer_version": self.scorer_version,
+            "mode": self.mode,
+            "overall": self.overall.to_dict(),
+            "by_source": {k: v.to_dict() for k, v in sorted(self.by_source.items())},
+            "by_category": {k: v.to_dict() for k, v in sorted(self.by_category.items())},
+            "four_way": {
+                bucket: {"count": n, "rate": round(n / total, 4)}
+                for bucket, n in sorted(self.four_way.items())
+            },
+            "all_null_floor": round(self.all_null_floor, 4),
+        }
+
+
+def _add(strata: dict[str, Stratum], key: str, r: FieldResult) -> None:
+    s = strata.setdefault(key, Stratum())
+    s.fields += 1
+    s.passed += int(r.passed)
+    s.weighted += r.weighted_score
+    s.four_way[r.bucket] += 1
+
+
+def build_report(docs: list[DocResult], mode: str = "unspecified") -> Report:
+    rep = Report(mode=mode)
+    empty_expected = 0
+    total = 0
+    doc_counts_source: dict[str, int] = defaultdict(int)
+    doc_counts_cat: dict[str, int] = defaultdict(int)
+
+    for doc in docs:
+        doc_counts_source[doc.source] += 1
+        doc_counts_cat[doc.category] += 1
+        for r in doc.fields:
+            total += 1
+            rep.overall.fields += 1
+            rep.overall.passed += int(r.passed)
+            rep.overall.weighted += r.weighted_score
+            rep.four_way[r.bucket] += 1
+            _add(rep.by_source, doc.source, r)
+            _add(rep.by_category, doc.category, r)
+            if is_empty(r.expected):
+                empty_expected += 1
+
+    rep.overall.docs = len(docs)
+    for src, n in doc_counts_source.items():
+        rep.by_source.setdefault(src, Stratum()).docs = n
+    for cat, n in doc_counts_cat.items():
+        rep.by_category.setdefault(cat, Stratum()).docs = n
+
+    # All-null floor: an empty extractor scores exactly the empty-GT fields
+    # (they land in `correct_absence`); everything else becomes a miss.
+    rep.all_null_floor = empty_expected / total if total else 0.0
+    return rep

fieldbench/cli.py

@@ -0,0 +1,134 @@
+"""FieldBench CLI: score prediction files against the corpus."""
+
+from __future__ import annotations
+
+import argparse
+import importlib
+import json
+import sys
+from pathlib import Path
+
+from .aggregate import build_report
+from .corpus import score_corpus
+from .run import run_corpus
+
+
+def _load_runner(ref: str):
+    """Load a runner from `module:attr`. attr is a Runner instance or a
+    zero-arg factory that returns one."""
+    if ":" not in ref:
+        raise ValueError(f"--runner must be 'module:attr', got {ref!r}")
+    mod_name, attr = ref.split(":", 1)
+    obj = getattr(importlib.import_module(mod_name), attr)
+    if hasattr(obj, "extract"):
+        return obj
+    return obj()  # factory
+
+
+def _print_table(report_dict: dict) -> None:
+    o = report_dict["overall"]
+    print(f"\nFieldBench  (scorer {report_dict['scorer_version']})")
+    print("=" * 52)
+    print(f"Overall:  {o['accuracy']:.1%}  ({o['passed']}/{o['fields']} fields, {o['docs']} docs)")
+    print(f"All-null floor: {report_dict['all_null_floor']:.1%}  <- a `{{}}`-emitting baseline scores this free")
+
+    print("\nBy source:")
+    for src, s in report_dict["by_source"].items():
+        print(f"  {src:<12} {s['accuracy']:.1%}  ({s['fields']} fields, {s['docs']} docs)")
+
+    print("\nFour-way outcomes:")
+    for bucket, v in report_dict["four_way"].items():
+        print(f"  {bucket:<16} {v['count']:>6}  ({v['rate']:.1%})")
+
+    print("\nBy category:")
+    for cat, s in sorted(report_dict["by_category"].items(), key=lambda kv: kv[1]["accuracy"]):
+        print(f"  {cat:<24} {s['accuracy']:.1%}  ({s['fields']} fields)")
+    print()
+
+
+def main(argv: list[str] | None = None) -> int:
+    parser = argparse.ArgumentParser(prog="fieldbench")
+    sub = parser.add_subparsers(dest="command", required=True)
+
+    score = sub.add_parser("score", help="Score prediction files against the corpus")
+    score.add_argument("--corpus", required=True, type=Path, help="Path to the corpus root")
+    score.add_argument("--results", required=True, type=Path, help="Directory of <stem>.json predictions")
+    score.add_argument("--category", default=None, help="Score a single category only")
+    score.add_argument(
+        "--mode", default="unspecified", help="Label for which representation the predictions came from (markdown/source/...)"
+    )
+    score.add_argument("--fuzzy-threshold", type=float, default=0.0, help="Off (0.0) for the official metric")
+    score.add_argument("--json", action="store_true", help="Emit the full report as JSON")
+    score.add_argument(
+        "--allow-missing",
+        action="store_true",
+        help="Do not fail when prediction files are missing (they are scored as all-null)",
+    )
+
+    run = sub.add_parser("run", help="Run an extractor over the corpus, writing per-doc predictions")
+    run.add_argument("--corpus", required=True, type=Path, help="Path to the corpus root")
+    run.add_argument("--out", required=True, type=Path, help="Output dir for <stem>.json predictions")
+    run.add_argument("--runner", required=True, help="Runner as 'module:attr' (instance or zero-arg factory)")
+    run.add_argument("--mode", default="markdown", help="Which representation to feed (markdown/text/source)")
+    run.add_argument("--category", default=None, help="Run a single category only")
+    run.add_argument("--limit", type=int, default=None, help="Stop after N docs (smoke testing)")
+    run.add_argument("--no-resume", action="store_true", help="Re-run docs even if a prediction file exists")
+
+    args = parser.parse_args(argv)
+
+    if args.command == "run":
+        if not args.corpus.is_dir():
+            print(f"error: corpus not found: {args.corpus}", file=sys.stderr)
+            return 2
+        try:
+            runner = _load_runner(args.runner)
+        except (ValueError, ImportError, AttributeError) as exc:
+            print(f"error: could not load runner {args.runner!r}: {exc}", file=sys.stderr)
+            return 2
+        stats = run_corpus(
+            args.corpus,
+            runner,
+            args.out,
+            mode=args.mode,
+            category=args.category,
+            resume=not args.no_resume,
+            limit=args.limit,
+            on_progress=lambda stem, status: print(f"  {stem}: {status}", file=sys.stderr),
+        )
+        print(
+            f"\nwrote {stats.written}, skipped {stats.skipped} (resume), "
+            f"errors {stats.errors}, no-representation {stats.no_representation}"
+        )
+        if stats.error_stems:
+            print(f"error docs: {', '.join(stats.error_stems[:10])}"
+                  f"{'…' if len(stats.error_stems) > 10 else ''}", file=sys.stderr)
+        return 1 if (stats.errors or stats.no_representation) else 0
+
+    if args.command == "score":
+        if not args.corpus.is_dir():
+            print(f"error: corpus not found: {args.corpus}", file=sys.stderr)
+            return 2
+        docs, missing = score_corpus(
+            args.corpus, args.results, category=args.category, fuzzy_threshold=args.fuzzy_threshold
+        )
+        if not docs:
+            print("error: no scorable documents found", file=sys.stderr)
+            return 2
+        report = build_report(docs, mode=args.mode).to_dict()
+        if args.json:
+            print(json.dumps(report, indent=2))
+        else:
+            _print_table(report)
+        if missing and not args.allow_missing:
+            print(
+                f"error: {missing} prediction file(s) missing — scored as all-null. "
+                f"Re-run with complete results, or pass --allow-missing to override.",
+                file=sys.stderr,
+            )
+            return 1
+        return 0
+    return 2
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

fieldbench/corpus.py

@@ -0,0 +1,167 @@
+"""Load the FieldBench corpus and score prediction files against it.
+
+Supports two on-disk layouts so scoring keeps working across the migration to
+the source-first structure (see corpus-structure.md):
+
+  Flat (current):
+    <category>/documents/<stem>.md
+    <category>/expected/<stem>.expected.json
+    <category>/manifests/<stem>.json
+
+  Source-first (target):
+    <category>/documents/<stem>/meta.json
+    <category>/documents/<stem>/repr/markdown.md   (+ other representations)
+    <category>/documents/<stem>/source.<ext>
+    <category>/expected/<stem>.expected.json        (GT unchanged, either layout)
+
+Predictions: a directory of `<stem>.json` files, each a flat {field: value} map
+(the zero-dependency format any system can emit). A missing prediction file is
+scored as all-null, never silently skipped.
+"""
+
+from __future__ import annotations
+
+import json
+from pathlib import Path
+
+import yaml
+
+from .aggregate import DocResult
+from .scoring import compare_field
+
+
+def _read_json(path: Path) -> dict | None:
+    try:
+        return json.loads(path.read_text())
+    except (OSError, json.JSONDecodeError):
+        return None
+
+
+def _load_manifest(cat_dir: Path, stem: str) -> dict | None:
+    """Dual-path manifest lookup: per-doc meta.json (new) → manifests/ (flat)."""
+    meta = cat_dir / "documents" / stem / "meta.json"
+    if meta.exists():
+        return _read_json(meta)
+    return _read_json(cat_dir / "manifests" / f"{stem}.json")
+
+
+def _source_of(manifest: dict) -> str:
+    """real | synthetic | unknown. Explicit `source` wins; else derive from format."""
+    src = str(manifest.get("source", "")).strip().lower()
+    if src in ("real", "synthetic"):
+        return src
+    fmt = str(manifest.get("source_format") or manifest.get("original_format", "")).lower()
+    if not fmt:
+        return "unknown"
+    return "synthetic" if "synth" in fmt else "real"
+
+
+def discover_categories(root: Path) -> list[str]:
+    cats = []
+    for entry in sorted(root.iterdir()):
+        if not entry.is_dir() or entry.name.startswith((".", "_")):
+            continue
+        if (entry / "documents").is_dir() and (entry / "expected").is_dir():
+            cats.append(entry.name)
+    return cats
+
+
+def _schema_mappings(root: Path, schema_ref: str | None, cache: dict) -> dict:
+    """{field_name: mappings} from a schema YAML (`fields.<name>.mappings`).
+
+    Enum-alias folding is what lets `10K/A` match GT `10-K/A`. Best-effort:
+    a missing/unreadable schema yields no mappings, never an error.
+    """
+    if not schema_ref:
+        return {}
+    if schema_ref in cache:
+        return cache[schema_ref]
+    out: dict = {}
+    path = root / schema_ref
+    try:
+        schema = yaml.safe_load(path.read_text())
+        for name, spec in (schema.get("fields") or {}).items():
+            if isinstance(spec, dict) and isinstance(spec.get("mappings"), dict):
+                out[name] = spec["mappings"]
+    except (OSError, yaml.YAMLError, AttributeError):
+        out = {}
+    cache[schema_ref] = out
+    return out
+
+
+def _iter_docs(root: Path, category: str):
+    cat = root / category
+    for expected_path in sorted((cat / "expected").glob("*.expected.json")):
+        stem = expected_path.name[: -len(".expected.json")]
+        expected = _read_json(expected_path)
+        manifest = _load_manifest(cat, stem)
+        if expected is None or manifest is None:
+            continue
+        yield stem, expected, manifest
+
+
+def representation_path(cat_dir: Path, stem: str, manifest: dict, mode: str) -> Path | None:
+    """Resolve the file a prediction generator should read for `mode`.
+
+    New layout: manifest.representations[mode].path (or source.* for mode='source').
+    Flat layout: documents/<stem>.md for mode in {markdown, text}.
+    """
+    doc_dir = cat_dir / "documents" / stem
+    if mode == "source":
+        artifact = manifest.get("source_artifact")
+        return (doc_dir / artifact) if artifact else None
+    reps = manifest.get("representations")
+    if isinstance(reps, dict) and mode in reps and isinstance(reps[mode], dict):
+        p = doc_dir / reps[mode]["path"]
+        return p if p.exists() else None
+    flat = cat_dir / "documents" / f"{stem}.md"  # flat layout has only markdown
+    return flat if (mode in ("markdown", "text") and flat.exists()) else None
+
+
+def list_documents(corpus_root: Path, mode: str = "markdown", category: str | None = None):
+    """Yield (stem, category, representation_path) for baseline/prediction runners."""
+    categories = [category] if category else discover_categories(corpus_root)
+    for cat in categories:
+        for stem, _expected, manifest in _iter_docs(corpus_root, cat):
+            yield stem, cat, representation_path(corpus_root / cat, stem, manifest, mode)
+
+
+def _load_prediction(results_dir: Path, stem: str) -> dict:
+    data = _read_json(results_dir / f"{stem}.json")
+    if isinstance(data, dict) and isinstance(data.get("fields"), dict):
+        return data["fields"]  # accept a {"fields": {...}} wrapper
+    return data if isinstance(data, dict) else {}
+
+
+def score_corpus(
+    corpus_root: Path,
+    results_dir: Path,
+    category: str | None = None,
+    fuzzy_threshold: float = 0.0,
+) -> tuple[list[DocResult], int]:
+    """Score every document. Returns (doc_results, missing_prediction_count).
+
+    Assert `missing_prediction_count == 0` before publishing any number — a
+    missing file is scored as all-null, not skipped, so it can't silently
+    inflate a system's accuracy.
+    """
+    categories = [category] if category else discover_categories(corpus_root)
+    docs: list[DocResult] = []
+    missing = 0
+    schema_cache: dict = {}
+
+    for cat in categories:
+        for stem, expected, manifest in _iter_docs(corpus_root, cat):
+            if not (results_dir / f"{stem}.json").exists():
+                missing += 1
+            prediction = _load_prediction(results_dir, stem)
+            mappings = _schema_mappings(corpus_root, manifest.get("schema"), schema_cache)
+            fields = [
+                compare_field(
+                    name, exp, prediction.get(name), fuzzy_threshold=fuzzy_threshold, mappings=mappings.get(name)
+                )
+                for name, exp in expected.items()
+            ]
+            docs.append(DocResult(stem, cat, _source_of(manifest), fields))
+
+    return docs, missing

fieldbench/run.py

@@ -0,0 +1,209 @@
+"""Drive an extractor over the corpus and write per-doc predictions.
+
+The other half of the benchmark loop: `fieldbench run` produces the `<stem>.json`
+prediction files that `fieldbench score` reads. Extractor-agnostic — supply any
+Runner (turns document text + schema into a {field: value} dict). A schema-driven
+LLM runner is provided; model-SDK wiring lives in examples/ so the core stays
+dependency-light. Runs are resumable (existing prediction files are skipped) and
+never invent results — a runner error writes nothing, so the doc is scored as
+all-null (a real miss), not silently dropped.
+"""
+
+from __future__ import annotations
+
+import json
+import re
+from dataclasses import dataclass, field
+from pathlib import Path
+from typing import Callable, Protocol
+
+import yaml
+
+from .corpus import _iter_docs, discover_categories, representation_path
+from .scoring import is_empty
+
+
+class Runner(Protocol):
+    """Anything that turns a document + its schema into a {field: value} dict."""
+
+    def extract(self, doc_text: str, schema: dict, stem: str) -> dict: ...
+
+
+@dataclass
+class RunStats:
+    written: int = 0
+    skipped: int = 0  # already present (resume)
+    errors: int = 0
+    no_representation: int = 0
+    error_stems: list = field(default_factory=list)
+
+
+def _schema_field_names(schema: dict) -> list[str]:
+    return list((schema.get("fields") or {}).keys())
+
+
+def build_extraction_prompt(doc_text: str, schema: dict) -> str:
+    """Generic schema-driven extraction prompt. No domain assumptions."""
+    lines = []
+    for name, spec in (schema.get("fields") or {}).items():
+        spec = spec if isinstance(spec, dict) else {}
+        typ = spec.get("type", "string")
+        desc = spec.get("description", "")
+        opts = spec.get("options")
+        suffix = f" (one of: {', '.join(map(str, opts))})" if isinstance(opts, list) and opts else ""
+        lines.append(f"- {name} ({typ}): {desc}{suffix}")
+    fields_block = "\n".join(lines)
+    return (
+        "Extract structured data from the document below.\n"
+        "Return ONLY a JSON object with EXACTLY these fields. "
+        "Use null when a field is not present in the document — do not guess.\n\n"
+        f"Fields:\n{fields_block}\n\n"
+        f"Document:\n{doc_text}\n\nJSON:"
+    )
+
+
+def parse_json_response(text: str) -> dict:
+    """Robustly pull a JSON object out of an LLM response."""
+    text = text.strip()
+    text = re.sub(r"^```(?:json)?\s*|\s*```$", "", text).strip()
+    try:
+        obj = json.loads(text)
+    except json.JSONDecodeError:
+        start, depth, end = text.find("{"), 0, -1
+        if start < 0:
+            return {}
+        for i in range(start, len(text)):
+            depth += text[i] == "{"
+            depth -= text[i] == "}"
+            if depth == 0:
+                end = i + 1
+                break
+        if end < 0:
+            return {}
+        try:
+            obj = json.loads(text[start:end])
+        except json.JSONDecodeError:
+            return {}
+    return obj if isinstance(obj, dict) else {}
+
+
+def window_text(text: str, size: int, overlap: int = 2000) -> list[str]:
+    """Split text into overlapping windows of `size` chars (overlap avoids
+    splitting a value across a boundary)."""
+    if len(text) <= size:
+        return [text]
+    step = max(1, size - overlap)
+    return [text[i : i + size] for i in range(0, len(text), step)]
+
+
+def merge_field_values(values: list) -> object:
+    """Merge one field's values across windows: concat+dedup lists, first
+    non-empty dict, else first non-empty scalar."""
+    non_empty = [v for v in values if not is_empty(v)]
+    if not non_empty:
+        return None
+    if any(isinstance(v, list) for v in non_empty):
+        out, seen = [], set()
+        for v in non_empty:
+            for item in v if isinstance(v, list) else [v]:
+                key = json.dumps(item, sort_keys=True, default=str)
+                if key not in seen:
+                    seen.add(key)
+                    out.append(item)
+        return out
+    for v in non_empty:
+        if isinstance(v, dict):
+            return v
+    return non_empty[0]
+
+
+class LLMRunner:
+    """Schema-driven runner over any `complete(prompt) -> str` callable.
+
+    Keeps model SDKs out of the core: pass a completion function (see
+    examples/). Output is filtered to the schema's declared fields.
+
+    `max_doc_chars` (None = off) makes it windowed: a document longer than the
+    budget is split into overlapping windows, each extracted, then merged
+    field-by-field — a fair long-document baseline that survives context limits.
+    """
+
+    def __init__(self, complete: Callable[[str], str], max_doc_chars: int | None = None, overlap: int = 2000):
+        self._complete = complete
+        self._max_doc_chars = max_doc_chars
+        self._overlap = overlap
+
+    def _extract_once(self, doc_text: str, schema: dict, allowed: set) -> dict:
+        parsed = parse_json_response(self._complete(build_extraction_prompt(doc_text, schema)))
+        return {k: v for k, v in parsed.items() if k in allowed} if allowed else parsed
+
+    def extract(self, doc_text: str, schema: dict, stem: str) -> dict:
+        allowed = set(_schema_field_names(schema))
+        if self._max_doc_chars and len(doc_text) > self._max_doc_chars:
+            windows = window_text(doc_text, self._max_doc_chars, self._overlap)
+            preds = [self._extract_once(w, schema, allowed) for w in windows]
+            names = allowed or {k for p in preds for k in p}
+            return {name: merge_field_values([p.get(name) for p in preds]) for name in names}
+        return self._extract_once(doc_text, schema, allowed)
+
+
+def _load_schema(root: Path, schema_ref: str | None, cache: dict) -> dict:
+    if not schema_ref:
+        return {}
+    if schema_ref in cache:
+        return cache[schema_ref]
+    try:
+        schema = yaml.safe_load((root / schema_ref).read_text()) or {}
+    except (OSError, yaml.YAMLError):
+        schema = {}
+    cache[schema_ref] = schema
+    return schema
+
+
+def run_corpus(
+    corpus_root: Path,
+    runner: Runner,
+    out_dir: Path,
+    mode: str = "markdown",
+    category: str | None = None,
+    resume: bool = True,
+    limit: int | None = None,
+    on_progress: Callable[[str, str], None] | None = None,
+) -> RunStats:
+    """Run `runner` over the corpus, writing out_dir/<stem>.json per doc."""
+    out_dir.mkdir(parents=True, exist_ok=True)
+    stats = RunStats()
+    schema_cache: dict = {}
+    categories = [category] if category else discover_categories(corpus_root)
+    done = 0
+
+    for cat in categories:
+        cat_dir = corpus_root / cat
+        for stem, _expected, manifest in _iter_docs(corpus_root, cat):  # _expected is GT — NOT used at run time
+            if limit is not None and done >= limit:
+                return stats
+            out_path = out_dir / f"{stem}.json"
+            if resume and out_path.exists():
+                stats.skipped += 1
+                continue
+            repr_path = representation_path(cat_dir, stem, manifest, mode)
+            if repr_path is None or not repr_path.exists():
+                stats.no_representation += 1
+                if on_progress:
+                    on_progress(stem, "no-representation")
+                continue
+            schema = _load_schema(corpus_root, manifest.get("schema"), schema_cache)
+            done += 1  # counts any attempted doc, so --limit works even when docs error
+            try:
+                prediction = runner.extract(repr_path.read_text(), schema, stem)
+                out_path.write_text(json.dumps(prediction, ensure_ascii=False, indent=2))
+                stats.written += 1
+                if on_progress:
+                    on_progress(stem, "ok")
+            except Exception as exc:  # noqa: BLE001 — a runner failure must not abort the whole run
+                stats.errors += 1
+                stats.error_stems.append(stem)
+                if on_progress:
+                    on_progress(stem, f"error: {exc}")
+
+    return stats

fieldbench/scoring.py

@@ -0,0 +1,333 @@
+"""Canonical field-level comparison for the FieldBench benchmark.
+
+This is the single source of truth for how an extracted value is compared to
+ground truth. It ports the semantics of Koji's ``cli/test_runner.py`` scorer
+(binary pass/fail + four-way null semantics, array F1, enum/mapping folding,
+punctuation-tolerant scalars) and makes the four-way outcome an explicit,
+first-class tag on every field so the aggregate can report a hallucination /
+miss / correct-absence breakdown, not just accuracy.
+
+Fully generic: no field names, document types, or domain knowledge.
+"""
+
+from __future__ import annotations
+
+import json
+import re
+from collections import Counter
+from dataclasses import dataclass
+from typing import Any, Literal
+
+Bucket = Literal["correct_absence", "hallucination", "miss", "match", "wrong_value"]
+
+SCORER_VERSION = "0.2.1"
+
+
+# ── Normalization helpers ─────────────────────────────────────────────
+
+
+def _levenshtein(a: str, b: str) -> int:
+    if len(a) < len(b):
+        return _levenshtein(b, a)
+    if not b:
+        return len(a)
+    prev = list(range(len(b) + 1))
+    for i, ca in enumerate(a):
+        curr = [i + 1]
+        for j, cb in enumerate(b):
+            curr.append(min(prev[j + 1] + 1, curr[j] + 1, prev[j] + (ca != cb)))
+        prev = curr
+    return prev[-1]
+
+
+# Common organizational designators. Folding trailing occurrences lets
+# "Microsoft Corp" match "Microsoft Corporation" and "3M Co" match "3M Company"
+# under the exact metric — a document prints the full legal form while an
+# authoritative index (e.g. EDGAR) stores an abbreviation.
+_ORG_SUFFIXES = {
+    "corp", "corporation", "co", "company", "inc", "incorporated", "ltd",
+    "limited", "llc", "llp", "lp", "plc", "pllc", "pc", "sa", "ag", "gmbh",
+    "nv", "bv", "ab", "oy", "spa",
+}
+
+
+def _fold_org_suffix(normalized: str) -> str:
+    """Strip trailing organizational designators from a punctuation-normalized
+    string (already lowercased, single-spaced). Returns the folded core."""
+    toks = normalized.split()
+    while toks and toks[-1] in _ORG_SUFFIXES:
+        toks.pop()
+    return " ".join(toks)
+
+
+def string_similarity(a: str, b: str) -> float:
+    """Levenshtein ratio in [0,1], case-insensitive after trimming."""
+    a = a.strip().lower()
+    b = b.strip().lower()
+    if a == b:
+        return 1.0
+    max_len = max(len(a), len(b))
+    if max_len == 0:
+        return 1.0
+    return 1.0 - _levenshtein(a, b) / max_len
+
+
+def normalize_date(value: Any) -> str | None:
+    """Normalize to YYYY-MM-DD, or None if not a recognizable date."""
+    if not isinstance(value, str):
+        return None
+    s = value.strip()
+    m = re.match(r"^(\d{4})-(\d{1,2})-(\d{1,2})$", s)
+    if m:
+        return f"{m.group(1)}-{int(m.group(2)):02d}-{int(m.group(3)):02d}"
+    m = re.match(r"^(\d{1,2})/(\d{1,2})/(\d{4})$", s)
+    if m:
+        return f"{m.group(3)}-{int(m.group(1)):02d}-{int(m.group(2)):02d}"
+    m = re.match(r"^(\d{1,2})[.\-](\d{1,2})[.\-](\d{4})$", s)
+    if m:
+        return f"{m.group(3)}-{int(m.group(1)):02d}-{int(m.group(2)):02d}"
+    return None
+
+
+def to_number(value: Any) -> float | None:
+    if isinstance(value, (int, float)) and not isinstance(value, bool):
+        return float(value)
+    if isinstance(value, str):
+        cleaned = value.replace("$", "").replace(",", "").strip()
+        try:
+            return float(cleaned)
+        except ValueError:
+            return None
+    return None
+
+
+def is_empty(value: Any) -> bool:
+    """None, empty string, empty list, empty dict → 'no value'."""
+    if value is None:
+        return True
+    if isinstance(value, str) and value.strip() == "":
+        return True
+    if isinstance(value, (list, dict)) and len(value) == 0:
+        return True
+    return False
+
+
+def _normalize_value(value: Any) -> Any:
+    if is_empty(value):
+        return None
+    if isinstance(value, bool):
+        return value
+    if isinstance(value, (int, float)):
+        return float(value)
+    if isinstance(value, str):
+        d = normalize_date(value)
+        if d is not None:
+            return d
+        n = to_number(value)
+        if n is not None:
+            return n
+        return value.strip().lower()
+    if isinstance(value, dict):
+        out: dict = {}
+        for k, v in value.items():
+            if isinstance(k, str) and k.startswith("__"):  # provenance keys
+                continue
+            nv = _normalize_value(v)
+            if nv is not None:
+                out[k] = nv
+        return out or None
+    if isinstance(value, list):
+        norm = [_normalize_value(v) for v in value]
+        norm = [v for v in norm if v is not None]
+        if not norm:
+            return None
+        if all(isinstance(v, dict) for v in norm):
+            norm.sort(key=lambda v: json.dumps(v, sort_keys=True))
+        return norm
+    return value
+
+
+def _normalize_for_set(items: list) -> list[str]:
+    keys = []
+    for item in items:
+        n = _normalize_value(item)
+        keys.append(json.dumps(n, sort_keys=True) if isinstance(n, dict) else json.dumps(n))
+    return keys
+
+
+def _dict_key_overlap(a: dict, b: dict) -> float:
+    na = _normalize_value(a)
+    nb = _normalize_value(b)
+    if not isinstance(na, dict) or not isinstance(nb, dict):
+        return 1.0 if json.dumps(na, sort_keys=True, default=str) == json.dumps(nb, sort_keys=True, default=str) else 0.0
+    all_keys = set(na) | set(nb)
+    if not all_keys:
+        return 1.0
+    matches = sum(
+        1
+        for k in all_keys
+        if json.dumps(na.get(k), sort_keys=True, default=str) == json.dumps(nb.get(k), sort_keys=True, default=str)
+    )
+    return matches / len(all_keys)
+
+
+def _array_of_dicts_similarity(expected: list, actual: list) -> float:
+    if not expected:
+        return 1.0 if not actual else 0.0
+    remaining = list(range(len(actual)))
+    total = 0.0
+    for exp_item in expected:
+        best, best_idx = 0.0, -1
+        for j in remaining:
+            sim = _dict_key_overlap(exp_item, actual[j])
+            if sim > best:
+                best, best_idx = sim, j
+        total += best
+        if best_idx >= 0:
+            remaining.remove(best_idx)
+    return total / len(expected)
+
+
+def array_f1(expected: list, actual: list) -> float:
+    """Element-wise F1 in [0,1] (quality-weighted precision/recall)."""
+    if not expected and not actual:
+        return 1.0
+    if not expected or not actual:
+        return 0.0
+    if all(isinstance(v, dict) for v in expected) and all(isinstance(v, dict) for v in actual):
+        recall = _array_of_dicts_similarity(expected, actual)
+        precision = _array_of_dicts_similarity(actual, expected)
+    else:
+        exp_c = Counter(_normalize_for_set(expected))
+        act_c = Counter(_normalize_for_set(actual))
+        matched = sum((exp_c & act_c).values())
+        recall = matched / len(expected)
+        precision = matched / len(actual)
+    if precision + recall == 0:
+        return 0.0
+    return 2 * precision * recall / (precision + recall)
+
+
+def resolve_mapping(value: str, mappings: dict) -> str:
+    """Fold a value to its canonical form via schema-declared enum aliases."""
+
+    def fold(s: object) -> str:
+        return re.sub(r"\s+", " ", str(s).strip().lower())
+
+    v = fold(value)
+    for canonical, aliases in mappings.items():
+        if v == fold(canonical):
+            return canonical
+        if isinstance(aliases, list) and any(v == fold(a) for a in aliases):
+            return canonical
+    return value
+
+
+# ── Result ────────────────────────────────────────────────────────────
+
+
+@dataclass
+class FieldResult:
+    field_name: str
+    passed: bool
+    bucket: Bucket
+    expected: Any = None
+    actual: Any = None
+    detail: str = ""
+    score: float | None = None  # explicit partial credit; None → derive from `passed`
+
+    @property
+    def weighted_score(self) -> float:
+        if self.score is not None:
+            return self.score
+        return 1.0 if self.passed else 0.0
+
+
+# ── Core comparison ───────────────────────────────────────────────────
+
+
+def compare_field(
+    field_name: str,
+    expected: Any,
+    actual: Any,
+    fuzzy_threshold: float = 0.0,
+    mappings: dict | None = None,
+) -> FieldResult:
+    """Compare one expected value against one extracted value.
+
+    Four-way null semantics run first, in order:
+      1. both empty            → correct_absence (PASS)
+      2. expected empty only   → hallucination   (FAIL)
+      3. actual empty only     → miss            (FAIL)
+      4. both present          → type-aware compare → match / wrong_value
+    """
+    exp_empty, act_empty = is_empty(expected), is_empty(actual)
+
+    if exp_empty and act_empty:
+        return FieldResult(field_name, True, "correct_absence", expected, actual, "correctly absent")
+    if exp_empty and not act_empty:
+        return FieldResult(field_name, False, "hallucination", expected, actual, f"hallucinated: expected null, got {actual!r}")
+    if act_empty:
+        return FieldResult(field_name, False, "miss", expected, actual, "missing from actual")
+
+    def matched(detail: str = "", score: float | None = None) -> FieldResult:
+        return FieldResult(field_name, True, "match", expected, actual, detail, score)
+
+    def wrong(detail: str, score: float | None = None) -> FieldResult:
+        return FieldResult(field_name, False, "wrong_value", expected, actual, detail, score)
+
+    # Date
+    ed, ad = normalize_date(expected), normalize_date(actual)
+    if ed is not None and ad is not None:
+        return matched() if ed == ad else wrong(f"expected {ed}, got {ad}")
+
+    # Number (absolute tolerance 0.01 — cent-exact by design)
+    en, an = to_number(expected), to_number(actual)
+    if en is not None and an is not None:
+        return matched() if round(abs(en - an), 10) <= 0.01 else wrong(f"expected {expected}, got {actual}")
+
+    # Array (element-wise F1; strict pass = exact-set, else fuzzy_threshold)
+    if isinstance(expected, list) and isinstance(actual, list):
+        f1 = array_f1(expected, actual)
+        exp_keys = sorted(_normalize_for_set(expected))
+        act_keys = sorted(_normalize_for_set(actual))
+        if exp_keys == act_keys:
+            return matched(score=1.0)
+        if fuzzy_threshold > 0 and expected:
+            matched_n = sum(1 for k in exp_keys if k in act_keys)
+            ratio = matched_n / len(exp_keys)
+            if ratio >= fuzzy_threshold:
+                return matched(f"fuzzy array match ({matched_n}/{len(exp_keys)})", score=f1)
+            if (
+                len(expected) == len(actual)
+                and all(isinstance(v, dict) for v in expected)
+                and all(isinstance(v, dict) for v in actual)
+                and _array_of_dicts_similarity(expected, actual) >= fuzzy_threshold
+            ):
+                return matched("fuzzy structural match", score=f1)
+        return wrong(f"array items differ ({f1:.0%} element F1)", score=f1)
+
+    # Scalar / string
+    if isinstance(expected, str) and isinstance(actual, str):
+        e, a = expected, actual
+        if mappings:
+            e, a = resolve_mapping(e, mappings), resolve_mapping(a, mappings)
+        if e.strip().lower() == a.strip().lower():
+            return matched()
+        ep = re.sub(r"[^a-z0-9]+", " ", e.lower()).strip()
+        ea = re.sub(r"[^a-z0-9]+", " ", a.lower()).strip()
+        if ep and ep == ea:
+            return matched()  # punctuation-only difference
+        ef, af = _fold_org_suffix(ep), _fold_org_suffix(ea)
+        if ef and ef == af:
+            # equal once trailing org designators (Corp/Corporation/Co/…) are folded;
+            # ef non-empty guards against two bare designators folding to nothing
+            return matched("organizational-suffix-equivalent")
+        if fuzzy_threshold > 0 and string_similarity(e, a) >= fuzzy_threshold:
+            return matched(f"fuzzy match ({string_similarity(e, a):.0%})")
+        return wrong(f"expected {expected!r}, got {actual!r}")
+
+    # Type mismatch or other → strict equality fallback
+    if _normalize_value(expected) == _normalize_value(actual):
+        return matched()
+    return wrong(f"expected {expected!r}, got {actual!r}")

fieldbench-0.2.1.dist-info/METADATA

@@ -0,0 +1,107 @@
+Metadata-Version: 2.4
+Name: fieldbench
+Version: 0.2.1
+Summary: A cross-domain, field-level benchmark for schema-driven document extraction
+Author: Frank Thomas
+License: MIT
+Project-URL: Homepage, https://github.com/fieldbench/fieldbench
+Project-URL: Corpus, https://github.com/fieldbench/corpus
+Keywords: benchmark,document-extraction,information-extraction,evaluation,nlp
+Classifier: Development Status :: 3 - Alpha
+Classifier: Intended Audience :: Science/Research
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Programming Language :: Python :: 3
+Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
+Requires-Python: >=3.10
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: pyyaml>=6
+Provides-Extra: dev
+Requires-Dist: pytest>=8; extra == "dev"
+Requires-Dist: ruff>=0.6; extra == "dev"
+Provides-Extra: openai
+Requires-Dist: openai>=1.0; extra == "openai"
+Provides-Extra: anthropic
+Requires-Dist: anthropic>=0.30; extra == "anthropic"
+Provides-Extra: llamaindex
+Requires-Dist: llama-index-core>=0.11; extra == "llamaindex"
+Requires-Dist: llama-index-llms-openai>=0.2; extra == "llamaindex"
+Provides-Extra: langchain
+Requires-Dist: langchain-openai>=0.2; extra == "langchain"
+Dynamic: license-file
+
+# FieldBench
+
+A cross-domain, field-level benchmark for **schema-driven document extraction** — and the canonical scorer that goes with it.
+
+Every document-extraction vendor claims 95%+ accuracy; almost none publish how they measure it. FieldBench makes extraction accuracy **falsifiable and comparable**: a shared corpus, a shared type-aware scorer, and a leaderboard anyone can submit to.
+
+> **Status: alpha (v0.1).** The scorer is the first piece to land. The corpus lives at [`fieldbench/corpus`](https://github.com/fieldbench/corpus). Leaderboard and HuggingFace packaging are in progress.
+
+## Install
+
+```bash
+pip install fieldbench   # once published; for now: pip install -e .
+```
+
+## Run a baseline
+
+`fieldbench run` drives any extractor over the corpus and writes the prediction files for you. It's extractor-agnostic — the whole integration surface is a `complete(prompt) -> str` callable (see `examples/openai_runner.py`):
+
+```bash
+pip install "fieldbench[openai]"
+OPENAI_API_KEY=... FIELDBENCH_MODEL=gpt-4o-mini \
+  fieldbench run --corpus ./corpus --out ./preds/gpt-4o-mini \
+    --runner examples.openai_runner:make_runner
+```
+
+Runs are **resumable** (existing predictions are skipped) and **never invent results** — a runner error writes nothing, so that document is scored as a real miss, not dropped.
+
+## Score your system
+
+Score prediction files — a flat `{field: value}` JSON named `<doc_id>.json` per document (produced by `fieldbench run` or by your own pipeline):
+
+```bash
+fieldbench score --corpus /path/to/corpus --results /path/to/predictions/
+```
+
+You get overall accuracy, the **all-null floor** (what an empty extractor scores for free), a **real-vs-synthetic** split, a **four-way outcome breakdown**, and a per-category table. `--json` emits the full report; `--category <name>` scopes to one category.
+
+## What makes the scoring type-aware
+
+`compare_field` runs **four-way null semantics** first — the thing plain accuracy hides:
+
+| Outcome | Meaning |
+|---|---|
+| `correct_absence` | field absent in GT, system correctly returned null |
+| `hallucination` | field absent in GT, system invented a value |
+| `miss` | field present in GT, system returned null |
+| `match` / `wrong_value` | both present → type-aware comparison |
+
+For present-vs-present it is tolerant where it should be and strict where it must be: numeric with cent-exact tolerance, date normalization, order-independent array F1 (partial credit), punctuation-insensitive strings, and schema enum-alias folding — but never masking a genuine content difference.
+
+## How to cite
+
+```bibtex
+@misc{fieldbench,
+  title  = {FieldBench: A Cross-Domain Benchmark for Schema-Driven Document Extraction},
+  author = {Thomas, Frank},
+  year   = {2026},
+  note   = {https://github.com/fieldbench}
+}
+```
+<!-- Replaced with the Zenodo DOI on release, and the paper citation once published. -->
+
+## Development
+
+```bash
+pip install -e ".[dev]"
+pytest        # golden scorer tests
+ruff check .
+```
+
+The golden tests in `tests/` are the anti-drift contract: they pin the canonical scoring semantics that any conforming implementation (including Koji's) must match.
+
+## License
+
+Code: MIT. The corpus is licensed per-source — see [`fieldbench/corpus`](https://github.com/fieldbench/corpus).

fieldbench-0.2.1.dist-info/RECORD

@@ -0,0 +1,12 @@
+fieldbench/__init__.py,sha256=Uecv5-tI8eUfS0sCqm588502nAYrgLRWTHW9atp1p4w,575
+fieldbench/aggregate.py,sha256=M4LymQ_Smw3QYPw4IUdFz-XPcot8U-Ai9yPXQy18rsk,4039
+fieldbench/cli.py,sha256=TF0VZkx8oFSCamK692mTQyBVCbt9iFKMZKaFtOFfFew,5724
+fieldbench/corpus.py,sha256=moljxHWUFPTTHpme8eEWH1hUz70iDqwYHvm3rUn-_EI,6341
+fieldbench/run.py,sha256=IMacKpXZ2Rc-rrHhfCVqJMiWfJBRoa8oOFC33n6gG_M,8022
+fieldbench/scoring.py,sha256=04EI2lBHYNa25lWiZZ6HTBfjoGYN4sfgXUoLZBrR_Nk,12491
+fieldbench-0.2.1.dist-info/licenses/LICENSE,sha256=AVla6qeNWJpcwjF6XMS3ONSL978yto6oWyt6H6FrAUc,1069
+fieldbench-0.2.1.dist-info/METADATA,sha256=zdgm7jXN5MFo0bEZ4dwI7P5nu0A2Y9mBpVKMcCfPyLg,4547
+fieldbench-0.2.1.dist-info/WHEEL,sha256=K260EYznzXsJYBQGqmI8VTxEdiZYNvDZwW9cBh9-_MA,91
+fieldbench-0.2.1.dist-info/entry_points.txt,sha256=qqwfIOAprmTIt1iJEO2VB6er7ov0CI537c0kGvzWWus,51
+fieldbench-0.2.1.dist-info/top_level.txt,sha256=cMN0klfFO_qUf3aHFLI5KfZhHI3W2AUC5EaNJEywing,11
+fieldbench-0.2.1.dist-info/RECORD,,

fieldbench-0.2.1.dist-info/WHEEL

@@ -0,0 +1,5 @@
+Wheel-Version: 1.0
+Generator: setuptools (83.0.0)
+Root-Is-Purelib: true
+Tag: py3-none-any
+

fieldbench-0.2.1.dist-info/entry_points.txt

@@ -0,0 +1,2 @@
+[console_scripts]
+fieldbench = fieldbench.cli:main

fieldbench-0.2.1.dist-info/licenses/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 Frank Thomas
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

fieldbench-0.2.1.dist-info/top_level.txt

@@ -0,0 +1 @@
+fieldbench