validbutwrong 0.1.0__py3-none-any.whl

validbutwrong/report.py

@@ -0,0 +1,139 @@
+"""Build the report: JSON (machine), markdown (human), SVG quadrant chart (README/CI artifact)."""
+
+from __future__ import annotations
+
+from html import escape
+from typing import Any
+
+from . import metrics as M
+from .tasks import Item, Task
+
+
+def comparisons(keys: list[str]) -> list[tuple[str, str, str]]:
+    """(from, to, role). The headline pairs toggle ONLY the grammar at a fixed output contract."""
+    out = []
+    for k in keys:
+        if k.startswith("constrained["):
+            out.append((k.replace("constrained[", "prompt_json[", 1), k, "headline: grammar off -> on, same contract"))
+    for k in keys:
+        if k.startswith("constrained[") or k.startswith("reason_then_constrain["):
+            out.append(("unconstrained", k, "secondary: also changes whether the model reasons in text (confounded)"))
+    return out
+
+
+def build(task: Task, items: list[Item], records: list[dict], meta: dict, mode: str) -> dict[str, Any]:
+    excluded = M.excluded_ids(records)
+    scored = [it for it in items if it.id not in set(excluded)]
+    states = M.score_run(task, items, records)
+    keys = list(states)
+    return {
+        "task": task.name, "correctness_label": task.correctness_label, "n": len(scored), "mode": mode,
+        "excluded": {"n": len(excluded), "ids": excluded,
+                     "reason": "the decoding backend could not compile the schema"},
+        "meta": meta,
+        "arms": {k: M.arm_metrics(states[k]) for k in keys},
+        "comparisons": [{**M.compare(states[a], states[b], a, b), "role": role}
+                        for a, b, role in comparisons(keys) if a in states and b in states],
+        "ledger": M.ledger(scored, states),
+    }
+
+
+def _pct(x: float | None) -> str:
+    return "n/a" if x is None else f"{100 * x:.1f}%"
+
+
+def _cell(m: dict | None) -> str:
+    if m is None:
+        return "n/a"
+    lo, hi = m["ci95"]
+    return f"{_pct(m['rate'])} [{_pct(lo)}, {_pct(hi)}]"
+
+
+def verdict_phrase(d: dict, n: int) -> str:
+    lo, hi = d["ci95"]
+    rng = f"95% CI [{100 * lo:+.1f}, {100 * hi:+.1f}] pts"
+    if not d["excludes_0"]:
+        return f"no detectable effect at N={n} ({rng})"
+    return f"{100 * d['delta']:+.1f} pts ({rng})"
+
+
+def markdown(r: dict) -> str:
+    n = r["n"]
+    meta = r["meta"]
+    lines = [f"# validbutwrong report: {r['task']} (N={n}, {r['mode']})", ""]
+    if r["mode"] == "replay":
+        lines.append(f"Scored from a committed replay generated {meta.get('started', '?')} in "
+                     f"{meta.get('wall_seconds', '?')} s on {meta.get('hardware', {}).get('cpu', '?')} "
+                     f"({meta.get('engine', {}).get('engine')}, {meta.get('engine', {}).get('model_file')}). "
+                     "Numbers are REPLAYED, not regenerated; use `--live` to regenerate.")
+        lines.append("")
+    if r["excluded"]["n"]:
+        lines += [f"**{r['excluded']['n']} item(s) excluded** from every arm ({r['excluded']['reason']}): "
+                  + ", ".join(r["excluded"]["ids"]), ""]
+    lines += ["## Metrics per arm (rate over all N items, 95% bootstrap CI)", "",
+              f"| arm | schema validity | {r['correctness_label']} | executable accuracy | wrong-but-valid |",
+              "|---|---|---|---|---|"]
+    for k, a in r["arms"].items():
+        lines.append(f"| `{k}` | {_cell(a['schema_validity'])} | {_cell(a['answer_accuracy'])} | "
+                     f"not computed in this run (task has no execution check) | {_cell(a['wrong_but_valid'])} |")
+    lines += ["", "## Paired comparisons (same items)", ""]
+    for c in r["comparisons"]:
+        f = c["flips"]
+        lines.append(f"**`{c['from']}` → `{c['to']}`** ({c['role']})")
+        for m, d in c["deltas"].items():
+            lines.append(f"- {m.replace('_', ' ')}: {verdict_phrase(d, n)}")
+        lines.append(f"- per-item flips: {f['right_to_wrong']} right→wrong, {f['wrong_to_right']} wrong→right")
+        lines.append("")
+    widths = [(k, a["answer_accuracy"]["ci95"][1] - a["answer_accuracy"]["ci95"][0]) for k, a in r["arms"].items()]
+    lines.append("CI widths (answer accuracy): " + ", ".join(f"`{k}` ±{50 * w:.1f} pts" for k, w in widths))
+    lines.append("")
+    return "\n".join(lines)
+
+
+def quadrant_svg(r: dict) -> str:
+    """One 2x2 panel per contract arm: rows valid/invalid, columns right/wrong. Zero dependencies."""
+    arms = [(k, a["quadrant"]) for k, a in r["arms"].items() if a["quadrant"]]
+    cell, gap, pad_top, pad_left = 64, 34, 64, 70
+    panel_w = 2 * cell + 20
+    width = pad_left + len(arms) * (panel_w + gap)
+    height = pad_top + 2 * cell + 64
+    n = r["n"]
+    ink, muted, grid, accent = "#1f2328", "#59636e", "#d0d7de", "#cf222e"
+    out = [f'<svg xmlns="http://www.w3.org/2000/svg" width="{width}" height="{height}" '
+           f'viewBox="0 0 {width} {height}" font-family="-apple-system, Segoe UI, Helvetica, Arial, sans-serif">',
+           f'<rect width="100%" height="100%" fill="#ffffff"/>',
+           f'<text x="{pad_left}" y="22" font-size="14" font-weight="600" fill="{ink}">'
+           f'{escape(r["task"])}: validity × correctness per arm (N={n})</text>',
+           f'<text x="{pad_left}" y="40" font-size="11" fill="{muted}">red cell = valid but wrong: '
+           f'the answers a 100%-valid dashboard hides</text>',
+           f'<text x="{pad_left - 8}" y="{pad_top + cell / 2 + 4}" font-size="11" fill="{muted}" text-anchor="end">valid</text>',
+           f'<text x="{pad_left - 8}" y="{pad_top + 1.5 * cell + 4}" font-size="11" fill="{muted}" text-anchor="end">invalid</text>']
+    for i, (k, q) in enumerate(arms):
+        x0 = pad_left + i * (panel_w + gap)
+        for row, v in enumerate(("valid", "invalid")):
+            for col, c in enumerate(("right", "wrong")):
+                cnt = q[f"{v}_{c}"]
+                share = cnt / n if n else 0
+                hot = v == "valid" and c == "wrong"
+                fill = accent if hot else "#57606a"
+                x, y = x0 + col * cell, pad_top + row * cell
+                out.append(f'<rect x="{x}" y="{y}" width="{cell - 2}" height="{cell - 2}" fill="{fill}" '
+                           f'fill-opacity="{0.08 + 0.82 * share:.3f}" stroke="{grid}"/>')
+                tc = "#ffffff" if share > 0.45 else ink
+                out.append(f'<text x="{x + cell / 2 - 1}" y="{y + cell / 2 + 5}" font-size="14" '
+                           f'font-weight="{600 if hot else 400}" text-anchor="middle" fill="{tc}">{cnt}</text>')
+        out.append(f'<text x="{x0 + cell / 2}" y="{pad_top - 8}" font-size="10" fill="{muted}" text-anchor="middle">right</text>')
+        out.append(f'<text x="{x0 + 1.5 * cell}" y="{pad_top - 8}" font-size="10" fill="{muted}" text-anchor="middle">wrong</text>')
+        out.append(f'<text x="{x0 + cell}" y="{pad_top + 2 * cell + 18}" font-size="11" fill="{ink}" '
+                   f'text-anchor="middle">{escape(k)}</text>')
+    out.append("</svg>")
+    return "\n".join(out)
+
+
+def gate(r: dict, threshold: float | None) -> list[str]:
+    """Arms whose wrong-but-valid rate exceeds the threshold (the CI exit-code gate)."""
+    if threshold is None:
+        return []
+    return [k for k, a in r["arms"].items()
+            if a["wrong_but_valid"] is not None and k.startswith("constrained[")
+            and a["wrong_but_valid"]["rate"] > threshold]

validbutwrong/run.py

@@ -0,0 +1,96 @@
+"""Live generation: run every item through every arm and write a replay file (JSONL).
+
+Line 1 is ``{"meta": {...}}`` (model file + sha256, backend versions, hardware, timings);
+each following line is one item's generations. Scoring never needs the model: ``score``
+re-reads this file, so a committed replay re-scores in seconds.
+"""
+
+from __future__ import annotations
+
+import hashlib
+import json
+import os
+import platform
+import subprocess
+import time
+from importlib import metadata
+
+from .arms import MAX_TOKENS, run_item
+from .engines import ENGINES
+from .tasks import Task
+
+
+def _version(pkg: str) -> str | None:
+    try:
+        return metadata.version(pkg)
+    except metadata.PackageNotFoundError:
+        return None
+
+
+def _sha256(path: str) -> str:
+    h = hashlib.sha256()
+    with open(path, "rb") as f:
+        for chunk in iter(lambda: f.read(1 << 20), b""):
+            h.update(chunk)
+    return h.hexdigest()
+
+
+def _hardware(n_threads: int) -> dict:
+    cpu = platform.processor()
+    if platform.system() == "Darwin":
+        try:
+            cpu = subprocess.run(["sysctl", "-n", "machdep.cpu.brand_string"],
+                                 capture_output=True, text=True).stdout.strip() or cpu
+        except OSError:
+            pass
+    return {"cpu": cpu, "machine": platform.machine(), "os": platform.platform(),
+            "logical_cpus": os.cpu_count(), "n_threads": n_threads}
+
+
+def generate(task: Task, model_path: str, out_path: str, n: int, engine: str = "outlines",
+             n_threads: int = 4, model_source: str | None = None, progress: bool = True) -> dict:
+    t_start = time.perf_counter()
+    wall_start = time.time()  # perf_counter pauses while the machine sleeps; time.time() does not
+    eng = ENGINES[engine](model_path, n_threads=n_threads)
+    t_st = time.perf_counter()
+    eng.selftest()  # refuses to run if the constraint does not bind
+    selftest_s = time.perf_counter() - t_st
+    items = task.items(n)
+    meta = {
+        "task": task.name, "n": len(items), "engine": eng.info(),
+        "model_sha256": _sha256(model_path), "model_source": model_source,
+        "versions": {p: _version(p) for p in ("validbutwrong", "outlines", "outlines-core",
+                                              "llama-cpp-python", "numba", "jsonschema")},
+        "hardware": _hardware(n_threads), "decoding": "greedy (temperature 0)",
+        "max_tokens": MAX_TOKENS, "selftest_seconds": round(selftest_s, 3),
+        "started": time.strftime("%Y-%m-%dT%H:%M:%S%z"),
+    }
+    os.makedirs(os.path.dirname(os.path.abspath(out_path)), exist_ok=True)
+    with open(out_path + ".partial", "w", encoding="utf-8") as f:
+        f.write(json.dumps({"meta": meta}) + "\n")
+        for i, it in enumerate(items):
+            t0 = time.perf_counter()
+            rec = run_item(eng, task, it)
+            rec["item_seconds"] = round(time.perf_counter() - t0, 3)
+            f.write(json.dumps(rec) + "\n")
+            f.flush()
+            if progress:
+                print(f"[{i + 1}/{len(items)}] {it.id} {rec['item_seconds']:.1f}s", flush=True)
+    meta["wall_seconds"] = round(time.perf_counter() - t_start, 1)            # compute time
+    meta["wall_clock_seconds"] = round(time.time() - wall_start, 1)          # includes any sleep
+    meta["ended"] = time.strftime("%Y-%m-%dT%H:%M:%S%z")
+    # rewrite the header with the final timings, then publish atomically
+    with open(out_path + ".partial", encoding="utf-8") as f:
+        lines = f.readlines()
+    lines[0] = json.dumps({"meta": meta}) + "\n"
+    with open(out_path, "w", encoding="utf-8") as f:
+        f.writelines(lines)
+    os.remove(out_path + ".partial")
+    return meta
+
+
+def read_replay(path: str) -> tuple[dict, list[dict]]:
+    with open(path, encoding="utf-8") as f:
+        meta = json.loads(f.readline())["meta"]
+        records = [json.loads(line) for line in f if line.strip()]
+    return meta, records

validbutwrong/schema.py

@@ -0,0 +1,66 @@
+"""JSON helpers: BFCL->JSON Schema conversion, first-object extraction, and validation."""
+
+from __future__ import annotations
+
+import json
+from typing import Any
+
+import jsonschema
+
+_BFCL_TYPES = {"dict": "object", "float": "number", "tuple": "array",
+               # "any" as an explicit union: some backends (outlines) cannot compile a typeless leaf
+               "any": ["string", "number", "integer", "boolean", "array", "object", "null"]}
+
+
+def bfcl_to_jsonschema(params: dict) -> dict:
+    """Convert BFCL's parameter dialect (``dict``/``float``/``tuple``/``any``) to JSON Schema.
+
+    BFCL schemas are not valid JSON Schema as published; a grammar compiler or ``jsonschema``
+    needs this conversion first. Descriptions and ``enum`` are kept; ``default`` is dropped
+    (it is documentation, not a constraint). Objects are closed (``additionalProperties: false``).
+    """
+    out: dict[str, Any] = {}
+    t = params.get("type")
+    if t is not None:
+        out["type"] = _BFCL_TYPES.get(t, t)
+    for key in ("description", "enum"):
+        if key in params:
+            out[key] = params[key]
+    if "properties" in params:
+        out["properties"] = {k: bfcl_to_jsonschema(v) for k, v in params["properties"].items()}
+        out["type"] = "object"
+        out["additionalProperties"] = False
+        if params.get("required"):
+            out["required"] = list(params["required"])
+    if "items" in params:
+        out["items"] = bfcl_to_jsonschema(params["items"])
+    return out
+
+
+def first_json_object(text: str) -> Any:
+    """Return the first parseable top-level JSON object in ``text``, or None.
+
+    Unlike a ``{.*}`` regex, this scans balanced braces and never rewrites the payload
+    (no whitespace stripping), so string values are graded as the model produced them.
+    """
+    dec = json.JSONDecoder()
+    i = text.find("{")
+    while i != -1:
+        try:
+            obj, _ = dec.raw_decode(text[i:])
+            if isinstance(obj, dict):
+                return obj
+        except json.JSONDecodeError:
+            pass
+        i = text.find("{", i + 1)
+    return None
+
+
+def is_valid(obj: Any, schema: dict) -> bool:
+    if obj is None:
+        return False
+    try:
+        jsonschema.validate(obj, schema)
+        return True
+    except jsonschema.ValidationError:
+        return False

validbutwrong/tasks.py

@@ -0,0 +1,277 @@
+"""Tasks: where items, prompts, gold answers and per-arm answer extraction live.
+
+A task supplies everything that is task-specific. The arms, engines, metrics and
+report never look inside an item; they only call the task's methods. Adding your
+own task means writing one of these classes (see ``Task``).
+"""
+
+from __future__ import annotations
+
+import ast
+import json
+import re
+from dataclasses import dataclass, field
+from typing import Any
+
+from .schema import bfcl_to_jsonschema
+
+
+@dataclass
+class Item:
+    id: str
+    prompt: str            # the user-facing problem text
+    gold: Any              # task-specific gold
+    meta: dict = field(default_factory=dict)
+
+
+class Task:
+    """Interface every task implements.
+
+    ``schemas``          name -> JSON Schema for the contract arms (prompt_json / constrained).
+    ``answer_schema``    which schema name reason_then_constrain packages into.
+    ``free_prompt``      instruction for the unconstrained arm (free text).
+    ``json_prompt``      instruction for the contract arms, given a schema.
+    ``extract_free``     free text -> answer (or None).
+    ``extract_json``     parsed JSON object -> answer (or None).
+    ``is_correct``       (answer, item) -> bool.
+    """
+
+    name = "task"
+    correctness_label = "answer accuracy"
+    schemas: dict[str, dict] = {}
+    answer_schema = "answer"
+
+    def items(self, n: int | None = None) -> list[Item]:
+        raise NotImplementedError
+
+    def schemas_for(self, item: Item) -> dict[str, dict]:
+        return self.schemas
+
+    def free_prompt(self, item: Item) -> str:
+        raise NotImplementedError
+
+    def json_prompt(self, item: Item, schema: dict) -> str:
+        raise NotImplementedError
+
+    def package_prompt(self, item: Item, schema: dict) -> str:
+        return ("Now give your final answer as ONLY a JSON object matching this JSON Schema, "
+                "with no other text:\n" + json.dumps(schema))
+
+    def extract_free(self, text: str, item: Item) -> Any:
+        raise NotImplementedError
+
+    def extract_json(self, obj: Any, item: Item) -> Any:
+        raise NotImplementedError
+
+    def is_correct(self, answer: Any, item: Item) -> bool:
+        raise NotImplementedError
+
+
+# --------------------------------------------------------------------------- GSM8K
+
+_NUM = re.compile(r"-?\d[\d,]*(?:\.\d+)?")
+
+
+def _to_number(x: Any) -> float | None:
+    if isinstance(x, bool):
+        return None
+    if isinstance(x, (int, float)):
+        return float(x)
+    if isinstance(x, str):
+        m = _NUM.findall(x.replace("$", ""))
+        if m:
+            try:
+                return float(m[-1].replace(",", ""))
+            except ValueError:
+                return None
+    return None
+
+
+class GSM8K(Task):
+    """GSM8K test split (openai/grade-school-math, MIT). Gold = the number after '####'."""
+
+    name = "gsm8k"
+    correctness_label = "answer accuracy"
+    schemas = {
+        "answer": {
+            "type": "object",
+            "properties": {"answer": {"type": "integer"}},
+            "required": ["answer"],
+            "additionalProperties": False,
+        },
+        "reasoning": {
+            "type": "object",
+            "properties": {"reasoning": {"type": "string"}, "answer": {"type": "integer"}},
+            "required": ["reasoning", "answer"],
+            "additionalProperties": False,
+        },
+    }
+    answer_schema = "answer"
+
+    def __init__(self, path: str):
+        self.path = path
+
+    def items(self, n: int | None = None) -> list[Item]:
+        out = []
+        with open(self.path, encoding="utf-8") as f:
+            for i, line in enumerate(f):
+                if n is not None and i >= n:
+                    break
+                row = json.loads(line)
+                gold = float(row["answer"].split("####")[-1].strip().replace(",", ""))
+                out.append(Item(id=f"gsm8k_test_{i}", prompt=row["question"], gold=gold))
+        return out
+
+    def free_prompt(self, item: Item) -> str:
+        return (f"{item.prompt}\n\nSolve the problem. Think step by step, then finish with a "
+                "final line of the form 'Answer: <number>'.")
+
+    def json_prompt(self, item: Item, schema: dict) -> str:
+        return (f"{item.prompt}\n\nSolve the problem. Respond with ONLY a JSON object matching "
+                f"this JSON Schema, with no other text:\n{json.dumps(schema)}")
+
+    def extract_free(self, text: str, item: Item) -> Any:
+        # Priority: an explicit "Answer: N" line, then LaTeX \boxed{N} (Qwen's habit), then the
+        # last number in the text. \boxed{} support was added after inspecting real outputs:
+        # without it, "\boxed{1400} bananas to last for 2 months" was read as 2.
+        m = re.findall(r"[Aa]nswer\s*[:=]\s*\**\s*\$?\s*(-?\d[\d,]*(?:\.\d+)?)", text)
+        if m:
+            return _to_number(m[-1])
+        boxed = re.findall(r"\\boxed\{([^{}]*)\}", text)
+        if boxed and _to_number(boxed[-1]) is not None:
+            return _to_number(boxed[-1])
+        return _to_number(text)  # fall back to the last number in the text
+
+    def extract_json(self, obj: Any, item: Item) -> Any:
+        if isinstance(obj, dict) and "answer" in obj:
+            return _to_number(obj["answer"])
+        return None
+
+    def is_correct(self, answer: Any, item: Item) -> bool:
+        return answer is not None and abs(answer - item.gold) < 1e-6
+
+
+# --------------------------------------------------------------------------- BFCL
+
+def _norm_str(s: str) -> str:
+    # BFCL-style string normalisation: case-insensitive, ignore spaces and common punctuation.
+    return re.sub(r"[ ,./\-_*^]", "", s.lower())
+
+
+def _value_matches(value: Any, allowed: Any) -> bool:
+    if isinstance(allowed, str):
+        return isinstance(value, str) and _norm_str(value) == _norm_str(allowed)
+    if isinstance(allowed, bool) or isinstance(value, bool):
+        return value is allowed
+    if isinstance(allowed, (int, float)):
+        return isinstance(value, (int, float)) and float(value) == float(allowed)
+    if isinstance(allowed, list):
+        return (isinstance(value, (list, tuple)) and len(value) == len(allowed)
+                and all(_value_matches(v, a) for v, a in zip(value, allowed)))
+    if isinstance(allowed, dict):
+        # nested dict in possible_answer: each key maps to a list of allowed values
+        return isinstance(value, dict) and ast_match(value, allowed, {"properties": dict.fromkeys(allowed)})
+    return value == allowed
+
+
+def ast_match(args: dict | None, gold: dict, params: dict) -> bool:
+    """Simplified BFCL-style AST match of one call's arguments against possible_answer.
+
+    Rules: every gold parameter must be present unless its allowed list contains "" (optional);
+    a present value must match one allowed value; no parameter outside the function's schema.
+    This mirrors the official checker's structure; it is NOT the official checker.
+    """
+    if not isinstance(args, dict):
+        return False
+    known = set(params.get("properties", {}))
+    if any(k not in known for k in args):
+        return False
+    for k, allowed in gold.items():
+        if k not in args:
+            if "" in allowed:
+                continue
+            return False
+        if not any(_value_matches(args[k], a) for a in allowed if a != ""):
+            return False
+    return True
+
+
+class BFCLSimple(Task):
+    """BFCL v3 'simple' category (gorilla-llm/Berkeley-Function-Calling-Leaderboard, apache-2.0).
+
+    One function per item. The contract arms emit the call's ARGUMENTS as a JSON object; the
+    schema is the function's parameter schema converted to JSON Schema. The unconstrained arm
+    writes a Python-syntax call, parsed with ``ast``. Scored by AST match, never by execution.
+    """
+
+    name = "bfcl-simple"
+    correctness_label = "AST-match accuracy"
+    answer_schema = "arguments"
+
+    def __init__(self, questions_path: str, answers_path: str):
+        self.questions_path = questions_path
+        self.answers_path = answers_path
+
+    def items(self, n: int | None = None) -> list[Item]:
+        with open(self.answers_path, encoding="utf-8") as f:
+            gold = {r["id"]: r["ground_truth"][0] for r in map(json.loads, f)}
+        out = []
+        with open(self.questions_path, encoding="utf-8") as f:
+            for i, line in enumerate(f):
+                if n is not None and i >= n:
+                    break
+                row = json.loads(line)
+                fn = row["function"][0]
+                (gname, gargs), = gold[row["id"]].items()
+                out.append(Item(id=row["id"], prompt=row["question"][0][-1]["content"],
+                                gold=gargs, meta={"function": fn, "gold_name": gname}))
+        return out
+
+    def schemas_for(self, item: Item) -> dict[str, dict]:
+        return {"arguments": bfcl_to_jsonschema(item.meta["function"]["parameters"])}
+
+    def _fn_doc(self, item: Item) -> str:
+        return json.dumps(item.meta["function"])
+
+    def free_prompt(self, item: Item) -> str:
+        fn = item.meta["function"]["name"]
+        return (f"You can call this function:\n{self._fn_doc(item)}\n\nRequest: {item.prompt}\n\n"
+                f"Write the single call to `{fn}` that fulfils the request, in Python syntax, "
+                f"e.g. {fn}(arg=value). Write only the call.")
+
+    def json_prompt(self, item: Item, schema: dict) -> str:
+        fn = item.meta["function"]["name"]
+        return (f"You can call this function:\n{self._fn_doc(item)}\n\nRequest: {item.prompt}\n\n"
+                f"Give the arguments for the call to `{fn}` as ONLY a JSON object matching this "
+                f"JSON Schema, with no other text:\n{json.dumps(schema)}")
+
+    def package_prompt(self, item: Item, schema: dict) -> str:
+        return ("Now give the arguments of that call as ONLY a JSON object matching this JSON "
+                "Schema, with no other text:\n" + json.dumps(schema))
+
+    def extract_free(self, text: str, item: Item) -> Any:
+        m = re.search(r"[\w.]+\s*\(.*\)", text.replace("\n", " "))
+        if not m:
+            return None
+        try:
+            call = ast.parse(m.group(0).strip(), mode="eval").body
+        except SyntaxError:
+            return None
+        if not isinstance(call, ast.Call):
+            return None
+        # Positional arguments are mapped by the function's declared parameter order. That is
+        # MORE lenient than BFCL's checker, deliberately: being charitable to the free-text
+        # baseline keeps any "the constraint helps" result conservative.
+        order = list(item.meta["function"]["parameters"].get("properties", {}))
+        try:
+            args = {order[i]: ast.literal_eval(a) for i, a in enumerate(call.args) if i < len(order)}
+            args.update({kw.arg: ast.literal_eval(kw.value) for kw in call.keywords if kw.arg})
+            return args
+        except (ValueError, SyntaxError):
+            return None
+
+    def extract_json(self, obj: Any, item: Item) -> Any:
+        return obj if isinstance(obj, dict) else None
+
+    def is_correct(self, answer: Any, item: Item) -> bool:
+        return ast_match(answer, item.gold, self.schemas_for(item)["arguments"])