evalpair 0.1.0__py3-none-any.whl

evalpair/__init__.py

@@ -0,0 +1,3 @@
+"""Run an LLM eval suite, cache every call, and compare two runs item by item with intervals."""
+
+__version__ = "0.1.0"

evalpair/__main__.py

@@ -0,0 +1,4 @@
+from .cli import main
+
+if __name__ == "__main__":
+    main()

evalpair/analysis.py

@@ -0,0 +1,141 @@
+"""Summaries of one run and comparisons of two, built from saved rows only (no model calls).
+
+A comparison pairs the two runs by item id over the items both answered (first pass). It reports the
+score difference with a paired bootstrap interval, how many items flipped each way, how often the answer
+changed at all, and, when a run was made with a repeat pass, that run's own answer-change rate: the noise
+floor. A change rate between two runs only means something above the floor.
+"""
+
+from __future__ import annotations
+
+from .stats import excludes_zero, mean, mean_ci
+
+EXCERPT = 400
+
+
+def _pp(x: float) -> str:
+    return f"{100 * x:+.1f} pp"
+
+
+def _pc(x: float) -> str:
+    return f"{100 * x:.1f}%"
+
+
+def _ci(ci: dict, pct=_pc) -> dict:
+    return {**{k: round(v, 4) for k, v in ci.items() if k != "n"}, "n": ci["n"],
+            "text": f"{pct(ci['mean'])} [{pct(ci['lo'])}, {pct(ci['hi'])}]" if ci["n"] else "no items"}
+
+
+def by_pass(rows: list[dict], p: int) -> dict[str, dict]:
+    return {r["id"]: r for r in rows if r["pass"] == p}
+
+
+def excerpt(text: str, n: int = EXCERPT) -> str:
+    text = text or ""
+    return text if len(text) <= n else "…" + text[-n:]
+
+
+def noise_floor(rows: list[dict]) -> dict | None:
+    """Share of items whose parsed answer changed between pass 1 and the repeat pass, and the score
+    change between them. None if the run had no repeat."""
+    a, b = by_pass(rows, 1), by_pass(rows, 2)
+    ids = [i for i in a if i in b]
+    if not ids:
+        return None
+    changed = [float(a[i]["answer"] != b[i]["answer"]) for i in ids]
+    groups = [a[i]["group"] for i in ids]
+    return {"items": len(ids), "answers_changed": int(sum(changed)),
+            "change_rate": _ci(mean_ci(changed, groups)),
+            "score_change": _ci(mean_ci([b[i]["score"] - a[i]["score"] for i in ids], groups), _pp)}
+
+
+def summarize(meta: dict, rows: list[dict], failures: int = 5) -> dict:
+    first = list(by_pass(rows, 1).values())
+    groups = [r["group"] for r in first]
+    wrong = [r for r in first if r["score"] < 1]
+    out = {
+        "run": meta["run"], "suite": meta["suite"], "model": meta["model"], "backend": meta["backend"],
+        "items": len(first), "score": _ci(mean_ci([r["score"] for r in first], groups)),
+        "no_answer": sum(r["answer"] is None for r in first), "truncated": sum(r["truncated"] for r in first),
+        "mean_output_tokens": round(mean(r["output_tokens"] for r in first), 1),
+        "noise_floor": noise_floor(rows),
+        "calls": meta.get("calls"), "cache_hits": meta.get("cache_hits"), "finished": meta.get("finished"),
+    }
+    if any(r["group"] is not None for r in first):
+        per = {}
+        for g in sorted({r["group"] for r in first}, key=str):
+            rs = [r for r in first if r["group"] == g]
+            per[str(g)] = {"items": len(rs), "score": round(mean(r["score"] for r in rs), 4)}
+        out["by_group"] = per
+    # unparsed and truncated replies first: they are usually the checker's or the budget's fault, not the model's
+    wrong.sort(key=lambda r: (r["answer"] is not None, not r["truncated"], r["id"]))
+    out["failures"] = [{"id": r["id"], "answer": r["answer"], "score": r["score"], "how": r["how"],
+                        "truncated": r["truncated"], "reply_end": excerpt(r["text"])} for r in wrong[:failures]]
+    return out
+
+
+def compare(meta_a: dict, rows_a: list[dict], meta_b: dict, rows_b: list[dict], examples: int = 5) -> dict:
+    a, b = by_pass(rows_a, 1), by_pass(rows_b, 1)
+    ids = [i for i in a if i in b]
+    notes = []
+    if meta_a.get("suite_hash") != meta_b.get("suite_hash"):
+        notes.append("The two runs used different versions of the suite (prompt, checker or items differ); "
+                     "only items with the same id are paired, and a difference may come from the suite.")
+    only = len(a) - len(ids), len(b) - len(ids)
+    if any(only):
+        notes.append(f"Paired on the {len(ids)} items both runs answered ({only[0]} only in A, {only[1]} only in B).")
+    if not ids:
+        return {"a": meta_a["run"], "b": meta_b["run"], "items": 0, "notes": notes + ["No items in common."]}
+    groups = [a[i]["group"] for i in ids]
+    sa, sb = [a[i]["score"] for i in ids], [b[i]["score"] for i in ids]
+    diff = mean_ci([y - x for x, y in zip(sa, sb)], groups)
+    worse = [i for i in ids if b[i]["score"] < a[i]["score"]]
+    better = [i for i in ids if b[i]["score"] > a[i]["score"]]
+    changed = [float(a[i]["answer"] != b[i]["answer"]) for i in ids]
+    out = {
+        "a": {"run": meta_a["run"], "model": meta_a["model"], "score": _ci(mean_ci(sa, groups))},
+        "b": {"run": meta_b["run"], "model": meta_b["model"], "score": _ci(mean_ci(sb, groups))},
+        "items": len(ids),
+        "difference": _ci(diff, _pp),
+        "worse_in_b": len(worse), "better_in_b": len(better),
+        "answers_changed": _ci(mean_ci(changed, groups)),
+    }
+    floors = {}
+    for name, rows in (("a", rows_a), ("b", rows_b)):
+        rep, p1 = by_pass(rows, 2), by_pass(rows, 1)
+        common = [i for i in ids if i in rep]
+        if common:
+            # per item: changed between the runs, minus changed between this run's own two passes
+            excess = [float(a[i]["answer"] != b[i]["answer"]) - float(p1[i]["answer"] != rep[i]["answer"])
+                      for i in common]
+            floors[name] = {"items": len(common),
+                            "change_rate": _ci(mean_ci([float(p1[i]["answer"] != rep[i]["answer"]) for i in common],
+                                                       [a[i]["group"] for i in common])),
+                            "excess_over_floor": _ci(mean_ci(excess, [a[i]["group"] for i in common]), _pp)}
+    out["noise_floor"] = floors or None
+    if excludes_zero(diff):
+        verdict = (f"B scores {'higher' if diff['mean'] > 0 else 'lower'} than A: "
+                   f"{_pp(diff['mean'])} [{_pp(diff['lo'])}, {_pp(diff['hi'])}], paired over {len(ids)} items.")
+    else:
+        verdict = (f"No clear difference in score: {_pp(diff['mean'])} [{_pp(diff['lo'])}, {_pp(diff['hi'])}], "
+                   f"an interval that includes zero, over {len(ids)} items.")
+    ch = out["answers_changed"]
+    verdict += f" Answers changed on {_pc(ch['mean'])} of items ({len(worse)} got worse, {len(better)} better)."
+    if floors:
+        # beyond noise only if it clears every floor measured, so a quiet run can't hide a noisy one
+        beyond = all(f["excess_over_floor"]["lo"] > 0 for f in floors.values())
+        rates = " and ".join(f"{_pc(f['change_rate']['mean'])} for {k.upper()}" for k, f in floors.items())
+        verdict += (f" The noise floor (the same model asked twice) is {rates}, so "
+                    + ("the change is beyond noise." if beyond else "the change is not clearly beyond noise."))
+    else:
+        verdict += " Neither run has a repeat pass, so there is no noise floor to compare against (run with repeat)."
+    out["verdict"] = verdict
+    out["notes"] = notes
+
+    def ex_rows(ids_):
+        return [{"id": i, "a_answer": a[i]["answer"], "b_answer": b[i]["answer"],
+                 "a_score": a[i]["score"], "b_score": b[i]["score"],
+                 "a_reply_end": excerpt(a[i]["text"], 240), "b_reply_end": excerpt(b[i]["text"], 240)}
+                for i in ids_[:examples]]
+    out["examples"] = {"worse_in_b": ex_rows(worse), "better_in_b": ex_rows(better)}
+    return out

evalpair/backends.py

@@ -0,0 +1,168 @@
+"""Model backends and the call cache.
+
+Two backends: Ollama's native /api/chat, and any OpenAI-compatible /v1/chat/completions server (vLLM,
+llama.cpp's server, LM Studio, or a hosted API). Every reply is cached in SQLite under a hash of
+(backend, base URL, model, messages, options, tag), so re-running a suite or re-scoring it costs nothing.
+`tag` is not sent to the model; it gives a repeat pass its own cache entry, so the repeat is a real second
+call and the difference between the two passes is the noise floor.
+"""
+
+from __future__ import annotations
+
+import hashlib
+import json
+import os
+import re
+import sqlite3
+import threading
+import time
+import urllib.error
+import urllib.request
+
+THINK_RE = re.compile(r"<think>.*?</think>", re.S)
+DEFAULT_URLS = {"ollama": "http://localhost:11434", "openai": "http://localhost:8000/v1"}
+
+
+class BackendError(RuntimeError):
+    pass
+
+
+class CacheMiss(RuntimeError):
+    pass
+
+
+class Cache:
+    def __init__(self, path: str):
+        os.makedirs(os.path.dirname(path) or ".", exist_ok=True)
+        self.lock = threading.Lock()
+        self.db = sqlite3.connect(path, check_same_thread=False, timeout=60)
+        self.db.execute("create table if not exists calls (key text primary key, model text, response text)")
+        self.db.commit()
+
+    def get(self, key: str) -> dict | None:
+        with self.lock:
+            row = self.db.execute("select response from calls where key = ?", (key,)).fetchone()
+        return json.loads(row[0]) if row else None
+
+    def put(self, key: str, model: str, resp: dict) -> None:
+        with self.lock:
+            self.db.execute("insert or replace into calls values (?, ?, ?)", (key, model, json.dumps(resp)))
+            self.db.commit()
+
+    def __len__(self) -> int:
+        with self.lock:
+            return self.db.execute("select count(*) from calls").fetchone()[0]
+
+    def close(self) -> None:
+        with self.lock:
+            self.db.close()
+
+
+def _post(url: str, body: dict, headers: dict, timeout: float) -> dict:
+    req = urllib.request.Request(url, json.dumps(body).encode(), {"Content-Type": "application/json", **headers})
+    with urllib.request.urlopen(req, timeout=timeout) as r:
+        return json.load(r)
+
+
+class Backend:
+    """One model on one server. `chat` returns {"text", "prompt_tokens", "output_tokens", "seconds",
+    "done_reason", "cached"}."""
+
+    def __init__(self, kind: str, model: str, cache: Cache, base_url: str | None = None,
+                 api_key: str | None = None, options: dict | None = None, think: bool | None = None,
+                 cache_only: bool = False, timeout: float = 600, retries: int = 4):
+        if kind not in DEFAULT_URLS:
+            raise BackendError(f"backend must be one of {sorted(DEFAULT_URLS)}")
+        self.kind, self.model, self.cache = kind, model, cache
+        self.base_url = (base_url or DEFAULT_URLS[kind]).rstrip("/")
+        self.api_key = api_key
+        self.options = {"temperature": 0, "seed": 7, **(options or {})}
+        self.think, self.cache_only, self.timeout, self.retries = think, cache_only, timeout, retries
+        self.stats = {"hits": 0, "calls": 0}
+        self.lock = threading.Lock()
+
+    def key(self, messages: list[dict], max_tokens: int, tag: str) -> str:
+        blob = json.dumps({"backend": self.kind, "url": self.base_url, "model": self.model, "messages": messages,
+                           "options": self.options, "max_tokens": max_tokens, "think": self.think, "tag": tag},
+                          sort_keys=True)
+        return hashlib.sha256(blob.encode()).hexdigest()
+
+    def body(self, messages: list[dict], max_tokens: int) -> tuple[str, dict, dict]:
+        o = self.options
+        if self.kind == "ollama":
+            body = {"model": self.model, "messages": messages, "stream": False,
+                    "options": {**o, "num_predict": max_tokens}}
+            if self.think is not None:
+                body["think"] = self.think
+            return self.base_url + "/api/chat", body, {}
+        body = {"model": self.model, "messages": messages, "max_tokens": max_tokens,
+                "temperature": o.get("temperature", 0)}
+        if "seed" in o:
+            body["seed"] = o["seed"]
+        if "top_p" in o:
+            body["top_p"] = o["top_p"]
+        headers = {"Authorization": f"Bearer {self.api_key}"} if self.api_key else {}
+        return self.base_url + "/chat/completions", body, headers
+
+    def parse(self, data: dict, wall: float) -> dict:
+        if self.kind == "ollama":
+            raw = data["message"]["content"]
+            return {"text": THINK_RE.sub("", raw).strip(), "raw": raw,
+                    "done_reason": data.get("done_reason", ""),
+                    "prompt_tokens": data.get("prompt_eval_count", 0), "output_tokens": data.get("eval_count", 0),
+                    # server-side compute time, which excludes time spent queued behind other clients
+                    "seconds": (data.get("prompt_eval_duration", 0) + data.get("eval_duration", 0)) / 1e9 or wall}
+        choice = data["choices"][0]
+        raw = choice["message"].get("content") or ""
+        usage = data.get("usage") or {}
+        return {"text": THINK_RE.sub("", raw).strip(), "raw": raw,
+                "done_reason": {"length": "length"}.get(choice.get("finish_reason"), choice.get("finish_reason") or ""),
+                "prompt_tokens": usage.get("prompt_tokens", 0), "output_tokens": usage.get("completion_tokens", 0),
+                "seconds": wall}
+
+    def chat(self, messages: list[dict], max_tokens: int, tag: str = "") -> dict:
+        key = self.key(messages, max_tokens, tag)
+        hit = self.cache.get(key)
+        if hit is not None:
+            with self.lock:
+                self.stats["hits"] += 1
+            return {**hit, "cached": True}
+        if self.cache_only:
+            raise CacheMiss(key)
+        url, body, headers = self.body(messages, max_tokens)
+        for attempt in range(self.retries + 1):
+            try:
+                t0 = time.time()
+                data = _post(url, body, headers, self.timeout)
+                wall = time.time() - t0
+                break
+            except urllib.error.HTTPError as e:
+                detail = e.read().decode(errors="replace")[:300]
+                # a request the server rejects won't succeed on retry; rate limits and 5xx might
+                if e.code < 500 and e.code != 429 or attempt == self.retries:
+                    raise BackendError(f"{self.kind} {self.model}: HTTP {e.code}: {detail}") from None
+            except (urllib.error.URLError, TimeoutError, ConnectionError) as e:
+                if attempt == self.retries:
+                    raise BackendError(f"{self.kind} at {self.base_url} unreachable: {e}") from None
+            time.sleep(min(2 ** attempt, 30))
+        try:
+            resp = self.parse(data, wall)
+        except (KeyError, IndexError, TypeError) as e:
+            raise BackendError(f"{self.kind} {self.model}: unexpected reply shape ({e!r})") from None
+        self.cache.put(key, self.model, resp)
+        with self.lock:
+            self.stats["calls"] += 1
+        return {**resp, "cached": False}
+
+    def describe(self) -> dict:
+        """What identifies the model, for the run record. For Ollama, the model digest."""
+        out = {"backend": self.kind, "model": self.model, "base_url": self.base_url}
+        if self.kind == "ollama":
+            try:
+                with urllib.request.urlopen(self.base_url + "/api/tags", timeout=30) as r:
+                    tags = json.load(r)
+                m = next((m for m in tags.get("models", []) if m.get("name") in (self.model, self.model + ":latest")), {})
+                out["digest"] = m.get("digest", "")
+            except (urllib.error.URLError, TimeoutError, ValueError):
+                out["digest"] = ""
+        return out

evalpair/checkers.py

@@ -0,0 +1,252 @@
+"""Deterministic answer checkers. Each returns a Check: the parsed answer, a score in [0, 1], and how the
+answer was found, so a wrong score can be traced to the parser rather than the model.
+
+The number and choice parsers come from the Measuring LLM Systems studies (prompt-brittleness, format-tax,
+groupthink, quant-cliff): an "Answer:" cue first, then fallbacks. A reply with no usable answer scores 0.
+"""
+
+from __future__ import annotations
+
+import json
+import re
+import unicodedata
+from dataclasses import dataclass
+from typing import Any, Callable
+
+NUM = r"-?\d[\d,]*(?:\.\d+)?"
+CUE = r"(?i:final answer|answer|a)"
+NULLS = {"", "null", "none", "n a", "na", "unknown", "not given", "not specified", "not provided"}
+
+
+@dataclass
+class Check:
+    answer: str | None
+    score: float
+    how: str
+
+
+def norm(s: Any) -> str:
+    """Casefolded, accents stripped, punctuation (all but £, as in the studies) to spaces, whitespace collapsed."""
+    s = unicodedata.normalize("NFKD", str(s))
+    s = "".join(ch for ch in s if not unicodedata.combining(ch)).casefold()
+    return re.sub(r"\s+", " ", re.sub(r"[^\w£ ]", " ", s)).strip()
+
+
+def _clean(text: str) -> str:
+    text = re.sub(r"<think>.*?</think>", "", text, flags=re.S)
+    return text.replace("**", "").replace("\\(", "").replace("\\)", "")
+
+
+def _join_groups(s: str) -> str:
+    """Read "9 500" (a space, no-break or thin space between thousands) as "9500". Used only on an
+    answer line or a \\boxed{}, where a space between digit groups is one number, not two."""
+    return re.sub(r"(?<=\d)[ \u00a0\u202f](?=\d{3}\b)", "", s)
+
+
+def canon_number(s: Any) -> str | None:
+    s = str(s).replace(",", "").replace("$", "").strip()
+    try:
+        x = float(s)
+    except ValueError:
+        return None
+    if x != x or x in (float("inf"), float("-inf")):
+        return None
+    if x == int(x):
+        return str(int(x))
+    return f"{x:.6g}"
+
+
+def parse_number(text: str) -> tuple[str | None, str]:
+    t = _clean(text)
+    cues = list(re.finditer(rf"(?m)^\s*[#>*\- ]*{CUE}\s*[:：]\s*(.*)$", t))
+    if cues:
+        m = re.search(NUM, _join_groups(cues[-1].group(1).replace("$", "")))
+        if m:
+            return canon_number(m.group(0)), "cue"
+    boxed = re.findall(r"\\boxed\{([^{}]*)\}", t)
+    if boxed:
+        m = re.search(NUM, _join_groups(boxed[-1]))
+        if m:
+            return canon_number(m.group(0)), "boxed"
+    nums = re.findall(NUM, t.replace("$", ""))
+    if nums:
+        return canon_number(nums[-1]), "last number"
+    return None, "none"
+
+
+def _letter_from_text(rest: str, options: list[str] | None, letters: str) -> str | None:
+    if not options:
+        return None
+    rest = norm(rest)
+    hits = [(len(norm(o)), i) for i, o in enumerate(options) if norm(o) and rest.startswith(norm(o))]
+    return letters[max(hits)[1]] if hits else None
+
+
+def parse_choice(text: str, options: list[str] | None = None, letters: str = "ABCD") -> tuple[str | None, str]:
+    t = _clean(text).strip()
+    L = f"[{letters}]"
+    cues = re.findall(rf"(?m)^\s*[#>*\- ]*{CUE}\s*[:：]\s*\(?({L})\b", t)
+    if cues:
+        return cues[-1], "cue"
+    m = re.search(rf"\b(?i:answer|option|choice) is\s*:?\s*\(?({L})\b", t)
+    if m:
+        return m.group(1), "answer is"
+    m = re.fullmatch(rf"\(?({L})\)?[.:)]?", t)
+    if m:
+        return m.group(1), "bare letter"
+    m = re.match(rf"\(?({L})[.):]\s", t + " ")
+    if m:
+        return m.group(1), "leading letter"
+    m = re.search(rf"\b(?i:answer)\s*[:：]?\s*\(?({L})\b", t)
+    if m:
+        return m.group(1), "inline cue"
+    cues = re.findall(rf"(?m)^\s*[#>*\- ]*{CUE}\s*[:：]\s*(.+)$", t)
+    if cues:
+        a = _letter_from_text(cues[-1], options, letters)
+        if a:
+            return a, "option text"
+    return None, "none"
+
+
+def _json_objects(t: str) -> list[str]:
+    """Top-level {...} spans, skipping braces inside strings."""
+    out, depth, start, in_str, esc = [], 0, None, False, False
+    for i, ch in enumerate(t):
+        if in_str:
+            if esc:
+                esc = False
+            elif ch == "\\":
+                esc = True
+            elif ch == '"':
+                in_str = False
+            continue
+        if ch == '"' and depth:
+            in_str = True
+        elif ch == "{":
+            if depth == 0:
+                start = i
+            depth += 1
+        elif ch == "}" and depth:
+            depth -= 1
+            if depth == 0:
+                out.append(t[start:i + 1])
+    return out
+
+
+def parse_json(text: str) -> tuple[dict | None, bool]:
+    """(object, strict). `strict` is True when the whole reply is exactly one JSON object; otherwise the
+    first JSON object anywhere in the reply, so code fences and a sentence around it are tolerated."""
+    t = _clean(text).strip()
+    try:
+        obj = json.loads(t)
+        if isinstance(obj, dict):
+            return obj, True
+    except json.JSONDecodeError:
+        pass
+    for blob in _json_objects(t):
+        try:
+            obj = json.loads(blob)
+        except json.JSONDecodeError:
+            continue
+        if isinstance(obj, dict):
+            return obj, False
+    return None, False
+
+
+def _is_null(x) -> bool:
+    return x is None or (isinstance(x, str) and norm(x) in NULLS)
+
+
+def _as_bool(x):
+    if isinstance(x, bool):
+        return x
+    s = norm(x)
+    if s in ("yes", "true", "y"):
+        return True
+    if s in ("no", "false", "n"):
+        return False
+    return "other"
+
+
+def field_ok(pred, gold) -> bool:
+    if gold is None:
+        return _is_null(pred)
+    if _is_null(pred):
+        return False
+    if isinstance(gold, bool):
+        return _as_bool(pred) == gold
+    if isinstance(gold, (int, float)):
+        return canon_number(pred) == canon_number(gold)
+    return norm(pred) == norm(gold)
+
+
+# ---------------------------------------------------------------- the checkers a suite can name
+
+def check_number(text: str, item: dict, cfg: dict) -> Check:
+    a, how = parse_number(text)
+    return Check(a, float(a is not None and a == canon_number(item["gold"])), how)
+
+
+def check_choice(text: str, item: dict, cfg: dict) -> Check:
+    letters = cfg.get("letters", "ABCD")
+    a, how = parse_choice(text, item.get("options"), letters)
+    return Check(a, float(a is not None and a == str(item["gold"]).strip().upper()), how)
+
+
+def check_exact(text: str, item: dict, cfg: dict) -> Check:
+    """The last non-empty line after any "Answer:" cue, compared after normalisation."""
+    t = _clean(text).strip()
+    cues = re.findall(rf"(?m)^\s*[#>*\- ]*{CUE}\s*[:：]\s*(.+)$", t)
+    if cues:
+        a, how = cues[-1].strip(), "cue"
+    else:
+        lines = [x for x in t.splitlines() if x.strip()]
+        a, how = (lines[-1].strip(), "last line") if lines else (None, "none")
+    golds = item["gold"] if isinstance(item["gold"], list) else [item["gold"]]
+    ok = a is not None and norm(a) in {norm(g) for g in golds}
+    return Check(a, float(ok), how)
+
+
+def check_contains(text: str, item: dict, cfg: dict) -> Check:
+    golds = item["gold"] if isinstance(item["gold"], list) else [item["gold"]]
+    t = norm(_clean(text))
+    hit = next((g for g in golds if norm(g) and f" {norm(g)} " in f" {t} "), None)
+    return Check(hit, float(hit is not None), "contains" if hit else "none")
+
+
+def check_regex(text: str, item: dict, cfg: dict) -> Check:
+    """The suite's `pattern` (first group, or the whole match), last match wins."""
+    pat = cfg.get("pattern")
+    if not pat:
+        raise ValueError("the regex checker needs a 'pattern' in the suite's [checker] table")
+    ms = list(re.finditer(pat, _clean(text), flags=re.M))
+    if not ms:
+        return Check(None, 0.0, "none")
+    m = ms[-1]
+    a = (m.group(1) if m.groups() else m.group(0)).strip()
+    golds = item["gold"] if isinstance(item["gold"], list) else [item["gold"]]
+    return Check(a, float(norm(a) in {norm(g) for g in golds}), "regex")
+
+
+def check_json_fields(text: str, item: dict, cfg: dict) -> Check:
+    """Share of the gold object's fields the reply gets right. A field whose gold is null must be null,
+    missing or a null-like string in the reply. Booleans accept yes/no and true/false."""
+    gold = item["gold"]
+    if not isinstance(gold, dict) or not gold:
+        raise ValueError(f"item {item.get('id')}: the json_fields checker needs a non-empty object as gold")
+    obj, strict = parse_json(text)
+    if obj is None:
+        return Check(None, 0.0, "no json")
+    ok = {f: field_ok(obj.get(f), g) for f, g in gold.items()}
+    answer = json.dumps({f: (None if _is_null(obj.get(f)) else str(obj.get(f))) for f in gold}, sort_keys=True)
+    return Check(answer, sum(ok.values()) / len(ok), "strict json" if strict else "json in text")
+
+
+CHECKERS: dict[str, Callable[[str, dict, dict], Check]] = {
+    "number": check_number,
+    "choice": check_choice,
+    "exact": check_exact,
+    "contains": check_contains,
+    "regex": check_regex,
+    "json_fields": check_json_fields,
+}