fieldbench 0.2.2__tar.gz → 0.2.3__tar.gz

{fieldbench-0.2.2/src/fieldbench.egg-info → fieldbench-0.2.3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: fieldbench
-Version: 0.2.2
+Version: 0.2.3
 Summary: A cross-domain, field-level benchmark for schema-driven document extraction
 Author: Frank Thomas
 License: MIT

{fieldbench-0.2.2 → fieldbench-0.2.3}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "fieldbench"
-version = "0.2.2"
+version = "0.2.3"
 description = "A cross-domain, field-level benchmark for schema-driven document extraction"
 readme = "README.md"
 requires-python = ">=3.10"

{fieldbench-0.2.2 → fieldbench-0.2.3}/src/fieldbench/__init__.py

@@ -5,7 +5,7 @@ from .corpus import list_documents, score_corpus
 from .run import LLMRunner, Runner, run_corpus
 from .scoring import SCORER_VERSION, FieldResult, compare_field
 
-__version__ = "0.2.2"
+__version__ = "0.2.3"
 
 __all__ = [
     "SCORER_VERSION",

{fieldbench-0.2.2 → fieldbench-0.2.3}/src/fieldbench/corpus.py

@@ -66,29 +66,44 @@ def discover_categories(root: Path) -> list[str]:
     return cats
 
 
-def _schema_mappings(root: Path, schema_ref: str | None, cache: dict) -> dict:
-    """{field_name: mappings} from a schema YAML (`fields.<name>.mappings`).
-
-    Enum-alias folding is what lets `10K/A` match GT `10-K/A`. Best-effort:
-    a missing/unreadable schema yields no mappings, never an error.
-    """
+def _load_schema(root: Path, schema_ref: str | None, cache: dict) -> dict:
+    """Parsed schema YAML (`{fields: {...}}`), cached. {} on missing/unreadable."""
     if not schema_ref:
         return {}
     if schema_ref in cache:
         return cache[schema_ref]
-    out: dict = {}
-    path = root / schema_ref
     try:
-        schema = yaml.safe_load(path.read_text())
-        for name, spec in (schema.get("fields") or {}).items():
-            if isinstance(spec, dict) and isinstance(spec.get("mappings"), dict):
-                out[name] = spec["mappings"]
-    except (OSError, yaml.YAMLError, AttributeError):
+        out = yaml.safe_load((root / schema_ref).read_text()) or {}
+    except (OSError, yaml.YAMLError):
         out = {}
     cache[schema_ref] = out
     return out
 
 
+def _schema_mappings(root: Path, schema_ref: str | None, cache: dict) -> dict:
+    """{field_name: mappings} from a schema YAML (`fields.<name>.mappings`).
+
+    Enum-alias folding is what lets `10K/A` match GT `10-K/A`. Best-effort:
+    a missing/unreadable schema yields no mappings, never an error.
+    """
+    schema = _load_schema(root, schema_ref, cache)
+    return {
+        name: spec["mappings"]
+        for name, spec in (schema.get("fields") or {}).items()
+        if isinstance(spec, dict) and isinstance(spec.get("mappings"), dict)
+    }
+
+
+def _schema_enums(root: Path, schema_ref: str | None, cache: dict) -> dict:
+    """{field_name: options} for enum-typed fields, for enum canonicalization."""
+    schema = _load_schema(root, schema_ref, cache)
+    return {
+        name: spec["options"]
+        for name, spec in (schema.get("fields") or {}).items()
+        if isinstance(spec, dict) and spec.get("type") == "enum" and isinstance(spec.get("options"), list)
+    }
+
+
 def _iter_docs(root: Path, category: str):
     cat = root / category
     for expected_path in sorted((cat / "expected").glob("*.expected.json")):
@@ -155,10 +170,16 @@ def score_corpus(
             if not (results_dir / f"{stem}.json").exists():
                 missing += 1
             prediction = _load_prediction(results_dir, stem)
-            mappings = _schema_mappings(corpus_root, manifest.get("schema"), schema_cache)
+            field_maps = _schema_mappings(corpus_root, manifest.get("schema"), schema_cache)
+            field_enums = _schema_enums(corpus_root, manifest.get("schema"), schema_cache)
             fields = [
                 compare_field(
-                    name, exp, prediction.get(name), fuzzy_threshold=fuzzy_threshold, mappings=mappings.get(name)
+                    name,
+                    exp,
+                    prediction.get(name),
+                    fuzzy_threshold=fuzzy_threshold,
+                    mappings=field_maps.get(name),
+                    enum_options=field_enums.get(name),
                 )
                 for name, exp in expected.items()
             ]

{fieldbench-0.2.2 → fieldbench-0.2.3}/src/fieldbench/scoring.py

@@ -20,7 +20,7 @@ from typing import Any, Literal
 
 Bucket = Literal["correct_absence", "hallucination", "miss", "match", "wrong_value"]
 
-SCORER_VERSION = "0.2.1"
+SCORER_VERSION = "0.2.2"
 
 
 # ── Normalization helpers ─────────────────────────────────────────────
@@ -221,6 +221,29 @@ def resolve_mapping(value: str, mappings: dict) -> str:
     return value
 
 
+def fold_enum(value: str, options: list) -> str:
+    """Canonicalize an enum value to the option contained within it.
+
+    Enum fields carry a fixed option set; a prediction is correct if it names
+    the right option, regardless of surrounding wording. E.g. for a
+    governing-law enum with option "New York", both "State of New York" and
+    "the laws of the State of New York" fold to "New York". Word-boundary
+    containment, longest option wins (so "North Dakota" beats "Dakota").
+    Returns the value unchanged if no option matches.
+    """
+    if not options:
+        return value
+    v = re.sub(r"\s+", " ", str(value).strip().lower())
+    best = None
+    for opt in options:
+        o = str(opt).strip().lower()
+        if o and re.search(r"\b" + re.escape(o) + r"\b", v) and (
+            best is None or len(o) > len(best[1])
+        ):
+            best = (opt, o)
+    return best[0] if best else value
+
+
 # ── Result ────────────────────────────────────────────────────────────
 
 
@@ -250,6 +273,7 @@ def compare_field(
     actual: Any,
     fuzzy_threshold: float = 0.0,
     mappings: dict | None = None,
+    enum_options: list | None = None,
 ) -> FieldResult:
     """Compare one expected value against one extracted value.
 
@@ -308,6 +332,8 @@ def compare_field(
     # Scalar / string
     if isinstance(expected, str) and isinstance(actual, str):
         e, a = expected, actual
+        if enum_options:
+            e, a = fold_enum(e, enum_options), fold_enum(a, enum_options)
         if mappings:
             e, a = resolve_mapping(e, mappings), resolve_mapping(a, mappings)
         if e.strip().lower() == a.strip().lower():

{fieldbench-0.2.2 → fieldbench-0.2.3/src/fieldbench.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: fieldbench
-Version: 0.2.2
+Version: 0.2.3
 Summary: A cross-domain, field-level benchmark for schema-driven document extraction
 Author: Frank Thomas
 License: MIT