extractscore 0.1.0__py3-none-any.whl

extractscore/__init__.py

@@ -0,0 +1,37 @@
+"""extractscore - field-level precision/recall/F1 for LLM-extracted JSON."""
+
+from __future__ import annotations
+
+__version__ = "0.1.0"
+
+from .config import Config, FieldRule
+from .diff import Diff, FieldDelta, Flip, diff
+from .errors import ConfigError, ExtractScoreError, InputError
+from .gate import GateFailure, check_gates
+from .io import ParseFailure, load_jsonl
+from .metrics import Counts, FieldStats
+from .score import DocResult, Report, compare, evaluate, evaluate_files
+
+__all__ = [
+    "Config",
+    "ConfigError",
+    "Counts",
+    "Diff",
+    "DocResult",
+    "ExtractScoreError",
+    "FieldDelta",
+    "FieldRule",
+    "FieldStats",
+    "Flip",
+    "GateFailure",
+    "InputError",
+    "ParseFailure",
+    "Report",
+    "__version__",
+    "check_gates",
+    "compare",
+    "diff",
+    "evaluate",
+    "evaluate_files",
+    "load_jsonl",
+]

extractscore/__main__.py

@@ -0,0 +1,5 @@
+import sys
+
+from .cli import main
+
+sys.exit(main())

extractscore/align.py

@@ -0,0 +1,111 @@
+"""Pairing items of a list: pure-Python Hungarian algorithm, no dependencies."""
+
+from __future__ import annotations
+
+import json
+from typing import Any, Callable, Sequence
+
+INF = float("inf")
+
+
+def hungarian(cost: Sequence[Sequence[float]]) -> list[int]:
+    """Minimum-cost assignment on a square matrix; returns ``col`` for every ``row``.
+
+    O(n^3) shortest-augmenting-path version.  Fine for the few hundred items a
+    document list usually has (200 items take a few tens of milliseconds).
+    """
+    n = len(cost)
+    u = [0.0] * (n + 1)
+    v = [0.0] * (n + 1)
+    owner = [0] * (n + 1)
+    way = [0] * (n + 1)
+    for i in range(1, n + 1):
+        owner[0] = i
+        j0 = 0
+        minv = [INF] * (n + 1)
+        used = [False] * (n + 1)
+        while True:
+            used[j0] = True
+            i0, delta, j1 = owner[j0], INF, 0
+            row = cost[i0 - 1]
+            for j in range(1, n + 1):
+                if not used[j]:
+                    current = row[j - 1] - u[i0] - v[j]
+                    if current < minv[j]:
+                        minv[j], way[j] = current, j0
+                    if minv[j] < delta:
+                        delta, j1 = minv[j], j
+            for j in range(n + 1):
+                if used[j]:
+                    u[owner[j]] += delta
+                    v[j] -= delta
+                else:
+                    minv[j] -= delta
+            j0 = j1
+            if owner[j0] == 0:
+                break
+        while j0:
+            j1 = way[j0]
+            owner[j0] = owner[j1]
+            j0 = j1
+    result = [0] * n
+    for j in range(1, n + 1):
+        result[owner[j] - 1] = j - 1
+    return result
+
+
+def match_items(
+    gold: Sequence[dict],
+    pred: Sequence[dict],
+    similarity: Callable[[dict, dict], float],
+    threshold: float,
+) -> tuple[list[tuple[int, int]], list[int], list[int]]:
+    """Pair gold items with predicted items regardless of order.
+
+    Identical items are paired first (cheap, and optimal).  The rest are paired by
+    maximising total similarity.  A pair whose similarity is below ``threshold``
+    (or zero) is dropped, so both items stay unmatched - a gold item with nothing
+    like it counts as missing and the stray predicted item as hallucinated, instead
+    of a misleading "wrong value".
+
+    Returns ``(pairs, unmatched_gold_indices, unmatched_pred_indices)``.
+    """
+    pairs: list[tuple[int, int]] = []
+    pool: dict[str, list[int]] = {}
+    for j, item in enumerate(pred):
+        pool.setdefault(_key(item), []).append(j)
+    rest_gold: list[int] = []
+    taken: set[int] = set()
+    for i, item in enumerate(gold):
+        candidates = pool.get(_key(item))
+        if candidates:
+            j = candidates.pop(0)
+            pairs.append((i, j))
+            taken.add(j)
+        else:
+            rest_gold.append(i)
+    rest_pred = [j for j in range(len(pred)) if j not in taken]
+
+    if rest_gold and rest_pred:
+        size = max(len(rest_gold), len(rest_pred))
+        sims = [[0.0] * size for _ in range(size)]
+        for a, i in enumerate(rest_gold):
+            for b, j in enumerate(rest_pred):
+                sims[a][b] = similarity(gold[i], pred[j])
+        assignment = hungarian([[1.0 - s for s in row] for row in sims])
+        matched_gold: set[int] = set()
+        matched_pred: set[int] = set()
+        for a, b in enumerate(assignment):
+            if a < len(rest_gold) and b < len(rest_pred):
+                s = sims[a][b]
+                if s > 0.0 and s >= threshold:
+                    pairs.append((rest_gold[a], rest_pred[b]))
+                    matched_gold.add(rest_gold[a])
+                    matched_pred.add(rest_pred[b])
+        rest_gold = [i for i in rest_gold if i not in matched_gold]
+        rest_pred = [j for j in rest_pred if j not in matched_pred]
+    return sorted(pairs), rest_gold, rest_pred
+
+
+def _key(item: Any) -> str:
+    return json.dumps(item, sort_keys=True, ensure_ascii=False, default=str)

extractscore/cli.py

@@ -0,0 +1,255 @@
+"""Command line: ``extractscore score | diff | demo``.
+
+Exit codes: 0 ok, 1 gate failed (``--min-f1``, ``--field-min``, ``--fail-on-regression``),
+2 usage or input error.
+"""
+
+from __future__ import annotations
+
+import argparse
+import json
+import re
+import shutil
+import sys
+import typing
+from pathlib import Path
+from typing import Sequence
+
+from . import __version__
+from .config import Config
+from .diff import diff as diff_reports
+from .errors import ExtractScoreError, InputError
+from .gate import check_gates
+from .i18n import LANGS, ref, resolve_lang, t
+from .render import diff_markdown, diff_text, report_markdown, report_text
+from .score import evaluate_files
+
+SAMPLE_DIR = Path(__file__).resolve().parent / "sample"
+
+
+def _early_lang(argv: Sequence[str]) -> str | None:
+    """Find ``--lang`` before the parser exists, so the help text itself can be translated."""
+    for i, arg in enumerate(argv):
+        if arg == "--lang" and i + 1 < len(argv):
+            return argv[i + 1]
+        if arg.startswith("--lang="):
+            return arg.split("=", 1)[1]
+    return None
+
+
+_KIND_WORDS = {"int": "int", "float": "float"}
+
+
+def _translate_argparse(message: str, lang: str) -> str:
+    """Translate argparse's own English messages (it has no translation hook)."""
+    if lang == "en":
+        return message
+    patterns = (
+        (r"the following arguments are required: (?P<names>.+)", "ap_required"),
+        (r"unrecognized arguments: (?P<names>.+)", "ap_unrecognized"),
+        (r"invalid choice: (?P<value>.+?) \(choose from (?P<choices>[^)]+)\)(?:, maybe you meant .+)?", "ap_invalid_choice"),
+        (r"invalid (?P<kind>\w+) value: (?P<value>.+)", "ap_invalid_value"),
+        (r"expected one argument", "ap_expected_one"),
+    )
+    named = re.fullmatch(r"argument (?P<name>[^:]+): (?P<rest>.+)", message, re.S)
+    if named:
+        return t(lang, "ap_argument", name=named["name"], message=_translate_argparse(named["rest"], lang))
+    for pattern, key in patterns:
+        found = re.fullmatch(pattern, message, re.S)
+        if found:
+            return t(lang, key, **found.groupdict())
+    return message
+
+
+def _make_parser_class(lang: str) -> type[argparse.ArgumentParser]:
+    """An ArgumentParser whose usage prefix, group titles and usage errors follow ``lang``."""
+
+    class Formatter(argparse.HelpFormatter):
+        def add_usage(self, usage, actions, groups, prefix=None):  # type: ignore[no-untyped-def]
+            super().add_usage(usage, actions, groups, t(lang, "usage_prefix") if prefix is None else prefix)
+
+    class Parser(argparse.ArgumentParser):
+        def __init__(self, *args, **kwargs):  # type: ignore[no-untyped-def]
+            kwargs.setdefault("formatter_class", Formatter)
+            kwargs["add_help"] = False
+            super().__init__(*args, **kwargs)
+            self._positionals.title = t(lang, "title_positional")
+            self._optionals.title = t(lang, "title_options")
+            self.add_argument("-h", "--help", action="help", default=argparse.SUPPRESS, help=t(lang, "help_help"))
+
+        def error(self, message: str) -> typing.NoReturn:
+            self.print_usage(sys.stderr)
+            self.exit(2, f"{self.prog}: {t(lang, 'ap_error')}: {_translate_argparse(message, lang)}\n")
+
+    return Parser
+
+
+def build_parser(lang: str) -> argparse.ArgumentParser:
+    """Build the argparse parser with all help text in ``lang``."""
+    parser_class = _make_parser_class(lang)
+    lang_parent = argparse.ArgumentParser(add_help=False)
+    lang_parent.add_argument("--lang", choices=LANGS, default=argparse.SUPPRESS, help=t(lang, "help_lang"))
+
+    parser = parser_class(prog="extractscore", description=t(lang, "help_description"), parents=[lang_parent])
+    parser.add_argument("--version", action="version", version=f"extractscore {__version__}", help=t(lang, "help_version"))
+    sub = parser.add_subparsers(dest="command", required=True, metavar="{score,diff,demo}", title=t(lang, "title_commands"), parser_class=parser_class)
+
+    score = sub.add_parser("score", help=t(lang, "help_score"), description=t(lang, "help_score"), parents=[lang_parent])
+    score.add_argument("gold", help=t(lang, "help_gold"))
+    score.add_argument("pred", help=t(lang, "help_pred"))
+    _common(score, lang)
+    score.add_argument("--min-f1", type=float, metavar="X", help=t(lang, "help_min_f1"))
+    score.add_argument("--field-min", action="append", default=[], metavar="PATH=X", help=t(lang, "help_field_min"))
+    score.add_argument("--top", type=int, default=5, metavar="N", help=t(lang, "help_top"))
+
+    diff = sub.add_parser("diff", help=t(lang, "help_diff"), description=t(lang, "help_diff"), parents=[lang_parent])
+    diff.add_argument("gold", help=t(lang, "help_gold"))
+    diff.add_argument("pred_a", metavar="A", help=t(lang, "help_pred_a"))
+    diff.add_argument("pred_b", metavar="B", help=t(lang, "help_pred_b"))
+    _common(diff, lang)
+    diff.add_argument("--fail-on-regression", action="store_true", help=t(lang, "help_fail_regress"))
+    diff.add_argument("--tolerance", type=float, default=0.0, metavar="X", help=t(lang, "help_tolerance"))
+
+    demo = sub.add_parser("demo", help=t(lang, "help_demo"), description=t(lang, "help_demo"), parents=[lang_parent])
+    demo.add_argument("--export", metavar="DIR", help=t(lang, "help_export"))
+    return parser
+
+
+def _common(p: argparse.ArgumentParser, lang: str) -> None:
+    p.add_argument("--config", metavar="FILE", help=t(lang, "help_config"))
+    p.add_argument("--id-key", default="id", metavar="KEY", help=t(lang, "help_id_key"))
+    p.add_argument("--format", choices=("table", "json", "md"), default="table", help=t(lang, "help_format"))
+    p.add_argument("--out", metavar="FILE", help=t(lang, "help_out"))
+
+
+def main(argv: Sequence[str] | None = None) -> int:
+    """Entry point; returns the process exit code."""
+    args_list = list(sys.argv[1:] if argv is None else argv)
+    lang = resolve_lang(_early_lang(args_list))
+    parser = build_parser(lang)
+    args = parser.parse_args(args_list)
+    lang = resolve_lang(getattr(args, "lang", None))
+    try:
+        if args.command == "score":
+            return _cmd_score(args, lang)
+        if args.command == "diff":
+            return _cmd_diff(args, lang)
+        return _cmd_demo(args, lang)
+    except ExtractScoreError as exc:
+        print(f"{t(lang, 'error_prefix')}: {exc.localized(lang)}", file=sys.stderr)
+        return 2
+
+
+# ------------------------------------------------------------------ commands
+def _load_config(path: str | None) -> Config:
+    return Config.from_file(path) if path else Config()
+
+
+def _parse_field_min(items: Sequence[str]) -> dict[str, float]:
+    result: dict[str, float] = {}
+    for item in items:
+        path, sep, value = item.rpartition("=")
+        try:
+            number = float(value)
+        except ValueError:
+            number = -1.0
+        if not sep or not path or not 0.0 <= number <= 1.0:
+            raise InputError("err_bad_option", detail=ref("d_field_min_format", item=item))
+        result[path] = number
+    return result
+
+
+def _emit(text: str, out: str | None, lang: str) -> None:
+    """Print to stdout, or write to ``--out`` and say so."""
+    if out is None:
+        sys.stdout.write(text)
+        return
+    try:
+        Path(out).write_text(text, encoding="utf-8")
+    except OSError as exc:
+        raise InputError("err_output_file", path=out, detail=str(exc)) from exc
+    print(t(lang, "wrote", path=out))
+
+
+def _cmd_score(args: argparse.Namespace, lang: str) -> int:
+    if args.min_f1 is not None and not 0.0 <= args.min_f1 <= 1.0:
+        raise InputError("err_bad_option", detail=ref("d_min_f1_range"))
+    if args.top < 0:
+        raise InputError("err_bad_option", detail=ref("d_top_negative"))
+    field_min = _parse_field_min(args.field_min)
+    report = evaluate_files(args.gold, args.pred, _load_config(args.config), id_key=args.id_key)
+    gated = args.min_f1 is not None or bool(field_min)
+    failures = check_gates(report, min_f1=args.min_f1, field_min=field_min)
+
+    if args.format == "json":
+        data = report.to_dict(args.top)
+        if gated:
+            data["gate"] = {
+                "passed": not failures,
+                "failures": [{"field": f.path, "f1": f.f1, "minimum": f.minimum} for f in failures],
+            }
+        text = json.dumps(data, indent=2, ensure_ascii=False) + "\n"
+    elif args.format == "md":
+        text = report_markdown(report, lang, args.top, failures, gated)
+    else:
+        text = report_text(report, lang, args.top, failures, gated)
+    _emit(text, args.out, lang)
+    if args.out and gated:  # keep the verdict visible in the CI log even when the report went to a file
+        for line in _gate_summary(failures, lang):
+            print(line)
+    return 1 if failures else 0
+
+
+def _gate_summary(failures, lang: str) -> list[str]:
+    lines = [t(lang, "gate_fail_line", path=f.path, f1=f"{f.f1:.3f}", minimum=f"{f.minimum:.3f}") for f in failures]
+    lines.append(t(lang, "gate_fail", n=len(failures)) if failures else t(lang, "gate_pass"))
+    return lines
+
+
+def _cmd_diff(args: argparse.Namespace, lang: str) -> int:
+    config = _load_config(args.config)
+    report_a = evaluate_files(args.gold, args.pred_a, config, id_key=args.id_key)
+    report_b = evaluate_files(args.gold, args.pred_b, config, id_key=args.id_key)
+    result = diff_reports(report_a, report_b, tolerance=args.tolerance)
+    if args.format == "json":
+        text = json.dumps(result.to_dict(), indent=2, ensure_ascii=False) + "\n"
+    elif args.format == "md":
+        text = diff_markdown(result, lang)
+    else:
+        text = diff_text(result, lang)
+    _emit(text, args.out, lang)
+    return 1 if args.fail_on_regression and result.regressions else 0
+
+
+def _cmd_demo(args: argparse.Namespace, lang: str) -> int:
+    if args.export:
+        return _export_demo(Path(args.export), lang)
+    config = Config.from_file(SAMPLE_DIR / "fields.toml")
+    gold = SAMPLE_DIR / "gold.jsonl"
+    report_a = evaluate_files(gold, SAMPLE_DIR / "pred_a.jsonl", config)
+    report_b = evaluate_files(gold, SAMPLE_DIR / "pred_b.jsonl", config)
+    print(t(lang, "demo_banner"))
+    print()
+    print(f"=== {t(lang, 'demo_run_a')}")
+    sys.stdout.write(report_text(report_a, lang))
+    print()
+    print(f"=== {t(lang, 'demo_run_b')}")
+    sys.stdout.write(diff_text(diff_reports(report_a, report_b), lang))
+    print()
+    print(t(lang, "demo_next"))
+    return 0
+
+
+def _export_demo(target: Path, lang: str) -> int:
+    if target.exists() and (not target.is_dir() or any(target.iterdir())):
+        raise InputError("err_export_exists", path=str(target))
+    shutil.copytree(SAMPLE_DIR, target, dirs_exist_ok=True, ignore=shutil.ignore_patterns("__pycache__"))
+    print(t(lang, "demo_banner"))
+    print(t(lang, "demo_exported", path=target))
+    print(f"  extractscore score {target}/gold.jsonl {target}/pred_a.jsonl --config {target}/fields.toml --min-f1 0.95")
+    print(f"  extractscore diff {target}/gold.jsonl {target}/pred_a.jsonl {target}/pred_b.jsonl --config {target}/fields.toml --fail-on-regression")
+    return 0
+
+
+if __name__ == "__main__":  # pragma: no cover
+    sys.exit(main())

extractscore/compare.py

@@ -0,0 +1,167 @@
+"""Typed value comparators.
+
+Each comparator answers one question: is the predicted value the same as the gold
+value, once format noise is removed?  The answer is one of three outcomes:
+
+* ``"match"``      - same value
+* ``"wrong"``      - a different value
+* ``"type_error"`` - the predicted value cannot be read as the field's type
+                     (``"N/A"`` in a number field, an object in a date field...)
+"""
+
+from __future__ import annotations
+
+import math
+import re
+from datetime import date, datetime
+from typing import Any, Sequence
+
+from .config import FieldRule
+
+MATCH = "match"
+WRONG = "wrong"
+TYPE_ERROR = "type_error"
+
+_CURRENCY = re.compile(r"usd|krw|eur|gbp|jpy|[$€£₩¥원\s,]", re.IGNORECASE)
+_TRUE = {"true", "yes", "y", "1"}
+_FALSE = {"false", "no", "n", "0"}
+
+
+def is_empty(value: Any, empty_as_null: bool = True) -> bool:
+    """True for ``None``; with ``empty_as_null`` also ``""``, ``[]`` and ``{}``."""
+    if value is None:
+        return True
+    if not empty_as_null:
+        return False
+    if isinstance(value, str):
+        return not value.strip()
+    if isinstance(value, (list, dict)):
+        return not value
+    return False
+
+
+def to_number(value: Any) -> float | None:
+    """Read a number from an int/float or a string such as ``"$1,250.00"`` or ``"12,000원"``."""
+    if isinstance(value, bool):
+        return None
+    if isinstance(value, (int, float)):
+        number = float(value)
+    elif isinstance(value, str):
+        cleaned = _CURRENCY.sub("", value)
+        try:
+            number = float(cleaned)
+        except ValueError:
+            return None
+    else:
+        return None
+    return number if math.isfinite(number) else None
+
+
+def to_date(value: Any, formats: Sequence[str]) -> date | None:
+    """Parse a date string with the first format that fits (order matters for ``11/03/2026``)."""
+    if not isinstance(value, str):
+        return None
+    text = value.strip()
+    for fmt in formats:
+        try:
+            return datetime.strptime(text, fmt).date()
+        except ValueError:
+            continue
+    return None
+
+
+def to_bool(value: Any) -> bool | None:
+    """Read a bool from a JSON bool or common strings (true/false/yes/no/1/0)."""
+    if isinstance(value, bool):
+        return value
+    if isinstance(value, (int, float)) and value in (0, 1):
+        return bool(value)
+    if isinstance(value, str):
+        lowered = value.strip().casefold()
+        if lowered in _TRUE:
+            return True
+        if lowered in _FALSE:
+            return False
+    return None
+
+
+def normalise_text(value: Any, casefold: bool = True) -> str:
+    """Collapse whitespace and (optionally) ignore case."""
+    text = re.sub(r"\s+", " ", str(value)).strip()
+    return text.casefold() if casefold else text
+
+
+def levenshtein(a: str, b: str) -> int:
+    """Edit distance between two strings."""
+    if a == b:
+        return 0
+    if not a:
+        return len(b)
+    if not b:
+        return len(a)
+    previous = list(range(len(b) + 1))
+    for i, char_a in enumerate(a, 1):
+        current = [i]
+        for j, char_b in enumerate(b, 1):
+            current.append(min(previous[j] + 1, current[j - 1] + 1, previous[j - 1] + (char_a != char_b)))
+        previous = current
+    return previous[-1]
+
+
+def text_similarity(a: str, b: str) -> float:
+    """1.0 for identical strings, falling with edit distance (normalised by the longer one)."""
+    longest = max(len(a), len(b))
+    return 1.0 if longest == 0 else 1.0 - levenshtein(a, b) / longest
+
+
+def _is_container(value: Any) -> bool:
+    return isinstance(value, (dict, list))
+
+
+def compare_values(rule: FieldRule, gold: Any, pred: Any, date_formats: Sequence[str]) -> str:
+    """Compare two non-empty scalar values under ``rule`` and return the outcome."""
+    kind = rule.type or "text"
+    if _is_container(pred):
+        return TYPE_ERROR
+
+    if kind == "number":
+        gold_n = to_number(gold)
+        if gold_n is None:  # unreadable gold: fall back to plain text equality
+            return MATCH if normalise_text(gold) == normalise_text(pred) else WRONG
+        pred_n = to_number(pred)
+        if pred_n is None:
+            return TYPE_ERROR
+        return MATCH if math.isclose(gold_n, pred_n, rel_tol=1e-9, abs_tol=max(rule.tol, 1e-12)) else WRONG
+
+    if kind == "date":
+        gold_d = to_date(gold, date_formats)
+        if gold_d is None:
+            return MATCH if normalise_text(gold) == normalise_text(pred) else WRONG
+        pred_d = to_date(pred, date_formats)
+        if pred_d is None:
+            return TYPE_ERROR
+        return MATCH if gold_d == pred_d else WRONG
+
+    if kind == "bool":
+        gold_b = to_bool(gold)
+        if gold_b is None:
+            return MATCH if normalise_text(gold) == normalise_text(pred) else WRONG
+        pred_b = to_bool(pred)
+        if pred_b is None:
+            return TYPE_ERROR
+        return MATCH if gold_b == pred_b else WRONG
+
+    if kind == "id":  # exact match; only surrounding whitespace is ignored
+        return MATCH if str(gold).strip() == str(pred).strip() else WRONG
+
+    if kind == "enum":
+        return MATCH if normalise_text(gold) == normalise_text(pred) else WRONG
+
+    # text
+    casefold = True if rule.casefold is None else rule.casefold
+    a, b = normalise_text(gold, casefold), normalise_text(pred, casefold)
+    if a == b:
+        return MATCH
+    if rule.fuzzy is not None and text_similarity(a, b) >= rule.fuzzy:
+        return MATCH
+    return WRONG