crossfoot 0.1.0__py3-none-any.whl

crossfoot/__init__.py

@@ -0,0 +1,29 @@
+"""crossfoot: fail-closed numeric grounding checks for LLM output."""
+from .extract import Mention, extract
+from .expressions import find_equations
+from .verify import (
+    ARITHMETIC_ERROR,
+    WRONG_ROW,
+    CONTRADICTED,
+    DERIVED,
+    EXACT,
+    PASSING,
+    ROUNDED,
+    UNCITED,
+    UNKNOWN_CITATION,
+    UNSUPPORTED,
+    WRONG_RECORD,
+    Checker,
+    Claim,
+    Report,
+    check,
+    load_sources,
+)
+
+__version__ = "0.1.0"
+
+__all__ = [
+    "check", "Checker", "extract", "load_sources", "Claim", "Report", "Mention", "PASSING",
+    "EXACT", "ROUNDED", "DERIVED", "WRONG_RECORD", "CONTRADICTED", "UNSUPPORTED", "UNCITED",
+    "UNKNOWN_CITATION", "ARITHMETIC_ERROR", "WRONG_ROW", "find_equations",
+]

crossfoot/cli.py

@@ -0,0 +1,116 @@
+"""Command line interface.
+
+    crossfoot check --output answer.txt --sources sources.json [--require-citations]
+    crossfoot eval cases.jsonl [--min-accuracy 1.0]
+
+``eval`` runs a labelled set of cases. Each JSONL line is::
+
+    {"id": "...", "output": "...", "sources": [...] | "sources_file": "x.json",
+     "expect": "pass" | "fail", "options": {"require_citations": true}}
+
+``expect`` defaults to "pass", so a set of real model outputs works as a CI gate.
+"""
+from __future__ import annotations
+
+import argparse
+import json
+import sys
+from pathlib import Path
+from typing import Any, Dict, List, Optional
+
+from .verify import check
+
+_OPTION_KEYS = {"require_citations", "allow_rounding", "allow_derived", "max_derivation_operands",
+                "share_scale_hints", "check_table_rows"}
+
+
+def _add_options(parser: argparse.ArgumentParser) -> None:
+    parser.add_argument("--require-citations", action="store_true",
+                        help="fail numbers in sentences that cite no source")
+    parser.add_argument("--strict", action="store_true",
+                        help="disallow rounding of directly quoted values")
+    parser.add_argument("--no-derived", action="store_true",
+                        help="disallow derived figures (%% change, share, sums, differences)")
+
+
+def _options(args: argparse.Namespace) -> Dict[str, Any]:
+    return {
+        "require_citations": args.require_citations,
+        "allow_rounding": not args.strict,
+        "allow_derived": not args.no_derived,
+    }
+
+
+def _read_json(path: Path) -> Any:
+    return json.loads(path.read_text(encoding="utf-8"))
+
+
+def _cmd_check(args: argparse.Namespace) -> int:
+    output = sys.stdin.read() if args.output == "-" else Path(args.output).read_text(encoding="utf-8")
+    report = check(output, _read_json(Path(args.sources)), **_options(args))
+    print(json.dumps(report.to_dict(), indent=2) if args.json else report.summary())
+    return 0 if report.passed else 1
+
+
+def _cmd_eval(args: argparse.Namespace) -> int:
+    cases_path = Path(args.cases)
+    lines = [l for l in cases_path.read_text(encoding="utf-8").splitlines() if l.strip()]
+    if not lines:
+        print(f"No cases found in {cases_path}")
+        return 1
+
+    agreed = 0
+    for number, line in enumerate(lines, start=1):
+        case = json.loads(line)
+        case_id = case.get("id", f"case-{number}")
+        if "sources_file" in case:
+            sources = _read_json(cases_path.parent / case["sources_file"])
+        else:
+            sources = case.get("sources", [])
+        options = {k: v for k, v in (case.get("options") or {}).items() if k in _OPTION_KEYS}
+        expect = case.get("expect", "pass")
+        if expect not in ("pass", "fail"):
+            raise ValueError(f"{case_id}: expect must be 'pass' or 'fail', got {expect!r}")
+
+        report = check(case["output"], sources, **options)
+        got = "pass" if report.passed else "fail"
+        ok = got == expect
+        agreed += ok
+        print(f"{'ok  ' if ok else 'MISS'} {case_id:<28} expected={expect:<4} got={got}")
+        if not ok or args.verbose:
+            print("     " + report.summary().replace("\n", "\n     "))
+
+    accuracy = agreed / len(lines)
+    print(f"\n{agreed}/{len(lines)} cases as expected (accuracy {accuracy:.2%}, "
+          f"threshold {args.min_accuracy:.2%})")
+    return 0 if accuracy >= args.min_accuracy else 1
+
+
+def main(argv: Optional[List[str]] = None) -> int:
+    if hasattr(sys.stdout, "reconfigure"):
+        sys.stdout.reconfigure(errors="replace")  # currency symbols on legacy consoles
+
+    parser = argparse.ArgumentParser(prog="crossfoot",
+                                     description="Fail-closed numeric grounding checks for LLM output.")
+    sub = parser.add_subparsers(dest="command", required=True)
+
+    p_check = sub.add_parser("check", help="check one output against its sources")
+    p_check.add_argument("--output", required=True, help="text file with the LLM output, or - for stdin")
+    p_check.add_argument("--sources", required=True, help="JSON file: list of strings or {id, text, fields}")
+    p_check.add_argument("--json", action="store_true", help="print a JSON report")
+    _add_options(p_check)
+    p_check.set_defaults(func=_cmd_check)
+
+    p_eval = sub.add_parser("eval", help="run a labelled JSONL case set (CI gate)")
+    p_eval.add_argument("cases", help="path to a .jsonl file")
+    p_eval.add_argument("--min-accuracy", type=float, default=1.0,
+                        help="fail if fewer than this fraction of cases match expectations")
+    p_eval.add_argument("-v", "--verbose", action="store_true", help="print every report")
+    p_eval.set_defaults(func=_cmd_eval)
+
+    args = parser.parse_args(argv)
+    return args.func(args)
+
+
+if __name__ == "__main__":
+    sys.exit(main())

crossfoot/expressions.py

@@ -0,0 +1,270 @@
+"""Find and re-check arithmetic written out in text.
+
+LLM answers often show their working::
+
+    (7167.1 - 6383.0) / 6383.0 * 100 = 12.28%
+    ($81 + $120) / 2 = $201 / 2 = $100.5 million
+    $7630.0 million - $6710.4 million = $918.2 million      <- wrong: 919.6
+
+``find_equations`` locates ``left = right`` pairs, evaluates the left side with
+a small recursive-descent parser (never ``eval``), and reports whether the
+stated right side agrees at its stated precision.
+"""
+from __future__ import annotations
+
+import re
+from dataclasses import dataclass, field
+from typing import List, Optional, Sequence, Tuple
+
+from .extract import mask_dates
+
+_SCALES = {"thousand": 1e3, "k": 1e3, "million": 1e6, "mn": 1e6, "m": 1e6, "billion": 1e9, "bn": 1e9}
+_UNIT_FACTORS = (1e3, 1e6, 1e9)
+_REL_EPS = 1e-9
+
+_TOKEN_RE = re.compile(
+    r"(?P<num>(?<![A-Za-z0-9_.])(?:HK\$|US\$|[$€£₹])?\s?"
+    r"(?P<digits>(?:\d{1,3}(?:,\d{3})+|\d+)(?:\.\d+)?)(?![\d]|\.\d)"
+    r"(?:\s?(?P<scale>thousand|million|billion|mn|bn|m|k)(?![A-Za-z]))?"
+    r"(?:\s?(?P<pct>%))?)"
+    r"|(?P<op>[+*/×÷]|[-−–]|\bx(?=\s)|\btimes\b)"
+    r"|(?P<lp>[(\[])"
+    r"|(?P<rp>[)\]])"
+    r"|(?P<eq>=|≈|~)"
+    r"|(?P<ws>\s+)",
+    re.IGNORECASE,
+)
+_ADD = {"+"}
+_SUB = {"-", "−", "–"}
+_MUL = {"*", "×", "x", "times"}
+_DIV = {"/", "÷"}
+
+
+@dataclass
+class Token:
+    kind: str  # num | op | lp | rp | eq | ws
+    text: str
+    start: int
+    end: int
+    value: float = 0.0
+    decimals: int = 0
+    scale: float = 1.0
+    has_scale: bool = False
+    percent: bool = False
+
+
+@dataclass
+class Equation:
+    left_value: float
+    left_span: Tuple[int, int]
+    left_tokens: List[Token]
+    right_value: float
+    right_tokens: List[Token]
+    right_number: Optional[Token]  # set when the right side is a single number
+    consistent: bool
+    operands: List[Token] = field(default_factory=list)
+
+
+def _make_token(m: "re.Match[str]") -> Token:
+    for kind in ("num", "op", "lp", "rp", "eq", "ws"):
+        if m.group(kind) is not None:
+            break
+    tok = Token(kind=kind, text=m.group(0), start=m.start(), end=m.end())
+    if kind == "num":
+        digits = m.group("digits").replace(",", "")
+        scale_word = (m.group("scale") or "").lower()
+        tok.decimals = len(digits.split(".", 1)[1]) if "." in digits else 0
+        tok.scale = _SCALES.get(scale_word, 1.0)
+        tok.has_scale = bool(scale_word)
+        tok.percent = m.group("pct") is not None
+        tok.value = float(digits) * tok.scale
+    # The span may include a leading space; extract() mentions start inside it.
+    tok.text = m.group(0).strip()
+    return tok
+
+
+def tokenize_runs(text: str) -> List[List[Token]]:
+    """Maximal runs of arithmetic tokens; any other character ends a run."""
+    text = mask_dates(text)
+    runs: List[List[Token]] = []
+    current: List[Token] = []
+    pos = 0
+    while pos < len(text):
+        m = _TOKEN_RE.match(text, pos)
+        if not m or m.end() == pos:
+            if current:
+                runs.append(current)
+                current = []
+            pos += 1
+            continue
+        tok = _make_token(m)
+        if tok.kind != "ws":
+            current.append(tok)
+        pos = m.end()
+    if current:
+        runs.append(current)
+    return runs
+
+
+class _Parser:
+    def __init__(self, tokens: Sequence[Token]) -> None:
+        self.tokens = tokens
+        self.i = 0
+
+    def _peek(self) -> Optional[Token]:
+        return self.tokens[self.i] if self.i < len(self.tokens) else None
+
+    def parse(self) -> Optional[float]:
+        value = self._expr()
+        if value is None or self.i != len(self.tokens):
+            return None
+        return value
+
+    def _expr(self) -> Optional[float]:
+        value = self._term()
+        while value is not None:
+            tok = self._peek()
+            if tok is None or tok.kind != "op" or tok.text not in _ADD | _SUB:
+                break
+            self.i += 1
+            right = self._term()
+            if right is None:
+                return None
+            value = value + right if tok.text in _ADD else value - right
+        return value
+
+    def _term(self) -> Optional[float]:
+        value = self._factor()
+        while value is not None:
+            tok = self._peek()
+            if tok is None or tok.kind != "op" or tok.text.lower() not in _MUL | _DIV:
+                break
+            self.i += 1
+            right = self._factor()
+            if right is None:
+                return None
+            if tok.text in _DIV:
+                if right == 0:
+                    return None
+                value = value / right
+            else:
+                value = value * right
+        return value
+
+    def _factor(self) -> Optional[float]:
+        tok = self._peek()
+        if tok is None:
+            return None
+        if tok.kind == "op" and tok.text in _SUB:
+            self.i += 1
+            inner = self._factor()
+            return None if inner is None else -inner
+        if tok.kind == "num":
+            self.i += 1
+            return tok.value
+        if tok.kind == "lp":
+            self.i += 1
+            inner = self._expr()
+            close = self._peek()
+            if inner is None or close is None or close.kind != "rp":
+                return None
+            self.i += 1
+            return inner
+        return None
+
+
+def _trim(tokens: List[Token]) -> List[Token]:
+    """Drop unmatched brackets at either end: '(a - b' / 'a - b)'."""
+    tokens = list(tokens)
+    while tokens and tokens[-1].kind == "rp" and \
+            sum(t.kind == "rp" for t in tokens) > sum(t.kind == "lp" for t in tokens):
+        tokens.pop()
+    while tokens and tokens[0].kind == "lp" and \
+            sum(t.kind == "lp" for t in tokens) > sum(t.kind == "rp" for t in tokens):
+        tokens.pop(0)
+    return tokens
+
+
+def _is_expression(tokens: Sequence[Token]) -> bool:
+    nums = sum(t.kind == "num" for t in tokens)
+    ops = sum(t.kind == "op" and not (t.text in _SUB and i == 0) for i, t in enumerate(tokens))
+    return nums >= 2 and ops >= 1
+
+
+def _evaluate(tokens: List[Token]) -> Optional[float]:
+    tokens = _trim(tokens)
+    if not _is_expression(tokens):
+        return None
+    return _Parser(tokens).parse()
+
+
+def _left_side(tokens: List[Token]) -> Optional[Tuple[List[Token], float]]:
+    """Longest suffix of the part before '=' that is a complete expression."""
+    for k in range(len(tokens)):
+        sub = _trim(tokens[k:])
+        value = _evaluate(sub)
+        if value is not None:
+            return sub, value
+    return None
+
+
+def _right_side(tokens: List[Token]) -> Optional[Tuple[List[Token], float, Optional[Token]]]:
+    """Longest prefix that is an expression, else the leading (signed) number."""
+    for k in range(len(tokens), 1, -1):
+        sub = _trim(tokens[:k])
+        value = _evaluate(sub)
+        if value is not None:
+            return sub, value, None
+    rest = [t for t in tokens if t.kind != "lp"]
+    if rest and rest[0].kind == "num":
+        return [rest[0]], rest[0].value, rest[0]
+    if len(rest) >= 2 and rest[0].kind == "op" and rest[0].text in _SUB and rest[1].kind == "num":
+        return rest[:2], -rest[1].value, rest[1]
+    return None
+
+
+def _agrees(left: float, left_tokens: Sequence[Token], right: float, right_number: Optional[Token]) -> bool:
+    candidates = [left]
+    left_scaled = any(t.kind == "num" and t.has_scale for t in left_tokens)
+    if right_number is not None:
+        tol = 0.5 * (10.0 ** -right_number.decimals) * right_number.scale
+        if right_number.percent:
+            candidates.append(left * 100)  # "(a / b) = 12.3%"
+        if right_number.has_scale and not left_scaled:
+            candidates.append(left * right_number.scale)  # "$201 / 2 = $100.5 million"
+        if left_scaled and not right_number.has_scale:
+            candidates.extend(left / f for f in _UNIT_FACTORS)
+    else:
+        tol = 0.005 * max(abs(left), abs(right))
+        candidates.extend([left * f for f in _UNIT_FACTORS] + [left / f for f in _UNIT_FACTORS])
+    return any(abs(c - right) <= tol + _REL_EPS * max(1.0, abs(right)) for c in candidates)
+
+
+def find_equations(text: str) -> List[Equation]:
+    """All ``left = right`` pairs whose left side is an arithmetic expression."""
+    equations: List[Equation] = []
+    for run in tokenize_runs(text):
+        parts: List[List[Token]] = [[]]
+        for tok in run:
+            if tok.kind == "eq":
+                parts.append([])
+            else:
+                parts[-1].append(tok)
+        for i in range(len(parts) - 1):
+            left = _left_side(parts[i])
+            right = _right_side(parts[i + 1])
+            if left is None or right is None:
+                continue
+            left_tokens, left_value = left
+            right_tokens, right_value, right_number = right
+            equations.append(Equation(
+                left_value=left_value,
+                left_span=(left_tokens[0].start, left_tokens[-1].end),
+                left_tokens=left_tokens,
+                right_value=right_value,
+                right_tokens=right_tokens,
+                right_number=right_number,
+                consistent=_agrees(left_value, left_tokens, right_value, right_number),
+                operands=[t for t in left_tokens if t.kind == "num"],
+            ))
+    return equations

crossfoot/extract.py

@@ -0,0 +1,184 @@
+"""Extract numeric mentions (amounts, percentages, years) from text.
+
+Every mention records the precision it was *stated* with, so later checks can
+tell legitimate rounding ("$3.2bn" for 3,214,000,000) from false precision
+("$3,214 million" when the source only says "$3.2bn").
+"""
+from __future__ import annotations
+
+import re
+from dataclasses import dataclass
+from typing import List, Optional
+
+_SCALES = {
+    "thousand": 1e3, "k": 1e3,
+    "lakh": 1e5,
+    "million": 1e6, "mn": 1e6, "m": 1e6,
+    "crore": 1e7, "cr": 1e7,
+    "billion": 1e9, "bn": 1e9, "b": 1e9,
+    "trillion": 1e12, "tn": 1e12,
+}
+
+_CURRENCIES = {
+    "$": "$", "us$": "USD", "usd": "USD",
+    "hk$": "HKD", "hkd": "HKD",
+    "€": "EUR", "eur": "EUR",
+    "£": "GBP", "gbp": "GBP",
+    "₹": "INR", "inr": "INR",
+}
+
+_NUMBER_RE = re.compile(
+    # "-$3,079" (sign attached). "$5,829 - $5,735" is subtraction, not a negative.
+    r"(?P<neg0>(?<![A-Za-z0-9])[-−](?=(?:HK\$|US\$|USD|HKD|EUR|GBP|INR|[$€£₹])))?"
+    r"(?:(?P<cur>HK\$|US\$|USD|HKD|EUR|GBP|INR|[$€£₹])\s?)?"
+    r"(?P<neg>(?<![A-Za-z0-9])[-−])?"
+    # Do not start inside identifiers such as FY2024, Q3, R1 or inside 3.5 / 3,200.
+    r"(?<![A-Za-z0-9_.,])"
+    r"(?P<num>\d{1,3}(?:,\d{2})+,\d{3}(?:\.\d+)?"   # Indian grouping: 1,23,456
+    r"|\d{1,3}(?:,\d{3})+(?:\.\d+)?"               # Western grouping: 1,234,567
+    r"|\d+(?:\.\d+)?)"
+    r"(?!\d|\.\d)"  # whole numbers only; not part of a version or IP: 8.4.1.1, 10.0.0.1
+    r"(?:\s?(?P<scale>trillion|billion|million|thousand|lakh|crore|bn|tn|mn|cr|k|m|b)(?![A-Za-z]))?"
+    r"(?:\s?(?P<pct>%|percentage points?|per\s?cent|percent|pp|bps|basis points?)(?![A-Za-z]))?",
+    re.IGNORECASE,
+)
+
+_LIST_MARKER_RE = re.compile(r"[.)](?:\s|$)")
+# "... settings. 2." / "steps: 1." - a step number after the end of a sentence.
+_AFTER_SENTENCE_END_RE = re.compile(r"(?:^|[.!?:;])\s*$")
+
+
+@dataclass(frozen=True)
+class Mention:
+    """A number found in text, normalised to a single float value."""
+
+    raw: str
+    value: float
+    kind: str  # "money" | "number" | "percent" | "pp" | "bps" | "year"
+    currency: Optional[str]
+    tolerance: float  # half a unit of the last stated digit, in value units
+    start: int
+    end: int
+    scaled: bool = False  # True if a scale word (k, m, bn, million...) was stated
+    scale: float = 1.0    # the stated scale factor (1e6 for "million")
+
+
+def _parse(match: "re.Match[str]") -> Mention:
+    num_text = match.group("num")
+    digits = num_text.replace(",", "")
+    decimals = len(digits.split(".", 1)[1]) if "." in digits else 0
+    base = float(digits)
+
+    scale_text = (match.group("scale") or "").lower()
+    scale = _SCALES.get(scale_text, 1.0)
+
+    cur_text = match.group("cur")
+    currency = _CURRENCIES.get(cur_text.lower()) if cur_text else None
+
+    pct_text = (match.group("pct") or "").lower()
+    if pct_text:
+        if pct_text.startswith(("bps", "basis")):
+            kind = "bps"
+        elif pct_text.startswith(("pp", "percentage")):
+            kind = "pp"
+        else:
+            kind = "percent"
+        scale = 1.0  # "12 m%" is not a thing; ignore any scale word
+    elif currency:
+        kind = "money"
+    elif not scale_text and decimals == 0 and "," not in num_text and 1900 <= base <= 2100:
+        kind = "year"
+    else:
+        kind = "number"
+
+    sign = -1.0 if (match.group("neg") or match.group("neg0")) else 1.0
+    return Mention(
+        raw=match.group(0).strip(),
+        value=sign * base * scale,
+        kind=kind,
+        currency=currency,
+        tolerance=0.5 * (10.0 ** -decimals) * scale,
+        start=match.start(),
+        end=match.end(),
+        scaled=bool(scale_text) and not pct_text,
+        scale=scale,
+    )
+
+
+# 11/30/04, 2019-06-30, 30.06.2019: dates, not quantities.
+_DATE_RE = re.compile(
+    r"(?<![\d/.-])(?:\d{1,2}/\d{1,2}/\d{2,4}|\d{4}-\d{2}-\d{2}|\d{1,2}\.\d{1,2}\.\d{4})(?![\d/.-]\d)"
+)
+
+
+def mask_dates(text: str) -> str:
+    """Replace numeric dates with spaces, keeping offsets unchanged."""
+    return _DATE_RE.sub(lambda m: " " * len(m.group(0)), text)
+
+
+_MONTH = (r"(?:jan(?:uary)?|feb(?:ruary)?|mar(?:ch)?|apr(?:il)?|may|june?|july?|aug(?:ust)?|"
+          r"sep(?:t(?:ember)?)?|oct(?:ober)?|nov(?:ember)?|dec(?:ember)?)")
+_MONTH_BEFORE_RE = re.compile(r"\b" + _MONTH + r"\.?\s*$", re.IGNORECASE)
+_MONTH_AFTER_RE = re.compile(r"\s*(?:st|nd|rd|th)?\s+(?:of\s+)?" + _MONTH + r"\b", re.IGNORECASE)
+_OBJECT = r"(?:(?:it|this|that|the\s+\w+)\s+)?"
+_MULTIPLY_BEFORE_RE = re.compile(
+    r"(?:[*×]|\bx|\btimes|\bmultipl(?:y|ied|ying)\s+" + _OBJECT + r"by)\s*$", re.IGNORECASE)
+_DIVIDE_BEFORE_RE = re.compile(
+    r"(?:/|÷|\bdivid(?:e|ed|ing)\s+" + _OBJECT + r"by)\s*$", re.IGNORECASE)
+_OPERATOR_AFTER_RE = re.compile(r"\s*[*×]")
+
+
+def _is_date_day(text: str, match: "re.Match[str]", mention: Mention) -> bool:
+    if mention.kind != "number" or mention.currency or mention.scaled or not (1 <= mention.value <= 31):
+        return False
+    if mention.value != int(mention.value):
+        return False
+    before = text[max(0, match.start() - 12):match.start()]
+    return bool(_MONTH_BEFORE_RE.search(before) or _MONTH_AFTER_RE.match(text, match.end()))
+
+
+def _is_formula_constant(text: str, match: "re.Match[str]", mention: Mention) -> bool:
+    """Arithmetic notation, not a claim: the 100 in "(a / b) * 100" or
+    "multiplying by 100", and the 2 in "/ 2" or "divide by 2"."""
+    if mention.kind != "number" or mention.currency or mention.scaled:
+        return False
+    value = mention.value
+    before = text[max(0, match.start() - 40):match.start()]
+    if value in (100.0, 1000.0):
+        return bool(_MULTIPLY_BEFORE_RE.search(before) or _OPERATOR_AFTER_RE.match(text, match.end()))
+    if value == int(value) and 2 <= value <= 12:
+        return bool(_DIVIDE_BEFORE_RE.search(before))
+    return False
+
+
+def extract(text: str) -> List[Mention]:
+    """Return all numeric mentions in ``text`` in order of appearance.
+
+    Skipped as non-claims: numbered-list markers ("1. ", "2) ") at the start
+    of a line, numbers inside identifiers (FY2024, Q3, R1), the day in dates
+    ("December 31, 2007", "31st March"), numeric dates ("11/30/04"),
+    version strings ("8.4.1.1") and formula constants ("* 100", "/ 2").
+    """
+    mentions: List[Mention] = []
+    text = mask_dates(text)
+    for match in _NUMBER_RE.finditer(text):
+        if not match.group("num"):
+            continue
+        line_start = text.rfind("\n", 0, match.start()) + 1
+        plain_marker = (
+            not (match.group("cur") or match.group("scale") or match.group("pct"))
+            and "." not in match.group("num") and "," not in match.group("num")
+            and _LIST_MARKER_RE.match(text, match.end()) is not None
+        )
+        is_list_marker = plain_marker and (
+            text[line_start:match.start()].strip() == ""
+            or (int(match.group("num")) <= 99
+                and _AFTER_SENTENCE_END_RE.search(text, max(0, match.start() - 3), match.start()) is not None)
+        )
+        if is_list_marker:
+            continue
+        mention = _parse(match)
+        if _is_date_day(text, match, mention) or _is_formula_constant(text, match, mention):
+            continue
+        mentions.append(mention)
+    return mentions

crossfoot/integrations/__init__.py

@@ -0,0 +1 @@
+"""Optional integrations with eval frameworks (DeepEval, promptfoo)."""

crossfoot/integrations/deepeval_metric.py

@@ -0,0 +1,51 @@
+"""DeepEval metric wrapper.
+
+    from deepeval import assert_test
+    from deepeval.test_case import LLMTestCase
+    from crossfoot.integrations.deepeval_metric import NumericGroundingMetric
+
+    case = LLMTestCase(input="...", actual_output=answer, retrieval_context=chunks)
+    assert_test(case, [NumericGroundingMetric(require_citations=True)])
+
+Sources are taken from ``retrieval_context`` (falling back to ``context``);
+chunk N can be cited as ``[N]`` or ``[SN]``.
+
+Requires ``pip install crossfoot[deepeval]``. Written against DeepEval's
+``BaseMetric`` interface; check it against the DeepEval version you use.
+"""
+from __future__ import annotations
+
+from typing import Any
+
+from deepeval.metrics import BaseMetric
+from deepeval.test_case import LLMTestCase
+
+from ..verify import check
+
+
+class NumericGroundingMetric(BaseMetric):
+    def __init__(self, threshold: float = 1.0, **check_options: Any) -> None:
+        self.threshold = threshold
+        self.check_options = check_options
+        self.score = 0.0
+        self.reason = ""
+        self.success = False
+        self.report = None
+
+    def measure(self, test_case: LLMTestCase, *args: Any, **kwargs: Any) -> float:
+        sources = list(test_case.retrieval_context or test_case.context or [])
+        self.report = check(test_case.actual_output or "", sources, **self.check_options)
+        self.score = self.report.score
+        self.success = self.report.passed and self.score >= self.threshold
+        self.reason = self.report.summary()
+        return self.score
+
+    async def a_measure(self, test_case: LLMTestCase, *args: Any, **kwargs: Any) -> float:
+        return self.measure(test_case)
+
+    def is_successful(self) -> bool:
+        return self.success
+
+    @property
+    def __name__(self) -> str:  # DeepEval reads the metric name from here
+        return "Numeric Grounding (crossfoot)"