toolcallbench-cn 0.1.0__py3-none-any.whl

tcb/__init__.py

@@ -0,0 +1 @@
+# -*- coding: utf-8 -*-

tcb/config.py

@@ -0,0 +1,161 @@
+# -*- coding: utf-8 -*-
+"""Central configuration: paths, pricing, budget caps, model list."""
+from __future__ import annotations
+
+import os
+from pathlib import Path
+
+REPO_ROOT = Path(__file__).resolve().parent.parent
+DATA_DIR = REPO_ROOT / "data"
+RESULTS_DIR = REPO_ROOT / "results"
+TASKS_DIR = REPO_ROOT / "tasks"
+
+WORLD_PATH = DATA_DIR / "world.json"
+COST_LEDGER = RESULTS_DIR / "cost_ledger.jsonl"
+BLOCKER_REPORT = REPO_ROOT / "BLOCKER_REPORT.md"
+
+DEEPSEEK_ENV = Path(
+    os.environ.get("TCB_ENV_FILE") or (REPO_ROOT / ".env")
+)
+BASE_URL = "https://api.deepseek.com"
+
+# Model registry: model_id -> provider endpoint, key env var, pricing.
+# Pricing = CNY per 1M tokens, ESTIMATES for the budget fuse only
+# (verify against vendor price lists; ledger records real token counts).
+MODEL_REGISTRY = {
+    "deepseek-chat": {
+        "base_url": "https://api.deepseek.com",
+        "key_env": ("DEEPSEEK_API_KEY",),
+        "pricing": {"input": 2.0, "output": 8.0},
+    },
+    "deepseek-reasoner": {
+        "base_url": "https://api.deepseek.com",
+        "key_env": ("DEEPSEEK_API_KEY",),
+        "pricing": {"input": 4.0, "output": 16.0},
+    },
+    # Aliyun Bailian / DashScope (OpenAI-compatible mode)
+    "qwen-max": {
+        "base_url": "https://dashscope.aliyuncs.com/compatible-mode/v1",
+        "key_env": ("DASHSCOPE_API_KEY", "BAILIAN_API_KEY"),
+        "pricing": {"input": 2.4, "output": 9.6},
+    },
+    "qwen-plus": {
+        "base_url": "https://dashscope.aliyuncs.com/compatible-mode/v1",
+        "key_env": ("DASHSCOPE_API_KEY", "BAILIAN_API_KEY"),
+        "pricing": {"input": 0.8, "output": 2.0},
+    },
+    "qwen-turbo": {
+        "base_url": "https://dashscope.aliyuncs.com/compatible-mode/v1",
+        "key_env": ("DASHSCOPE_API_KEY", "BAILIAN_API_KEY"),
+        "pricing": {"input": 0.3, "output": 0.6},
+    },
+    "qwen-flash": {
+        "base_url": "https://dashscope.aliyuncs.com/compatible-mode/v1",
+        "key_env": ("DASHSCOPE_API_KEY", "BAILIAN_API_KEY"),
+        "pricing": {"input": 0.0, "output": 0.0},
+    },
+    # --- Bailian "Token Plan" (套餐专属 sk-sp- key) models.
+    # The plan bills per CALL from a quota, not per token; pricing below are
+    # shadow estimates (list-price analogues) so the budget fuse still works.
+    # Endpoint pending verification with a valid plan key (probe_qwen.py).
+    "qwen3.5-plus": {
+        "base_url": "https://dashscope.aliyuncs.com/compatible-mode/v1",
+        "key_env": ("DASHSCOPE_API_KEY", "BAILIAN_API_KEY"),
+        "pricing": {"input": 0.8, "output": 2.0},
+    },
+    "qwen3.5-flash": {
+        "base_url": "https://dashscope.aliyuncs.com/compatible-mode/v1",
+        "key_env": ("DASHSCOPE_API_KEY", "BAILIAN_API_KEY"),
+        "pricing": {"input": 0.0, "output": 0.0},
+    },
+    "kimi-k3": {
+        "base_url": "https://dashscope.aliyuncs.com/compatible-mode/v1",
+        "key_env": ("DASHSCOPE_API_KEY", "BAILIAN_API_KEY"),
+        "pricing": {"input": 4.0, "output": 16.0},
+    },
+    "kimi-k2.6": {
+        "base_url": "https://dashscope.aliyuncs.com/compatible-mode/v1",
+        "key_env": ("DASHSCOPE_API_KEY", "BAILIAN_API_KEY"),
+        "pricing": {"input": 4.0, "output": 16.0},
+    },
+    "glm-5": {
+        "base_url": "https://dashscope.aliyuncs.com/compatible-mode/v1",
+        "key_env": ("DASHSCOPE_API_KEY", "BAILIAN_API_KEY"),
+        "pricing": {"input": 2.0, "output": 8.0},
+    },
+    "MiniMax-M3": {
+        "base_url": "https://dashscope.aliyuncs.com/compatible-mode/v1",
+        "key_env": ("DASHSCOPE_API_KEY", "BAILIAN_API_KEY"),
+        "pricing": {"input": 2.0, "output": 8.0},
+    },
+    "deepseek-v3.2": {
+        "base_url": "https://dashscope.aliyuncs.com/compatible-mode/v1",
+        "key_env": ("DASHSCOPE_API_KEY", "BAILIAN_API_KEY"),
+        "pricing": {"input": 2.0, "output": 8.0},
+    },
+    "mock": {"base_url": "", "key_env": (), "pricing": {"input": 0.0, "output": 0.0}},
+    # --- Local Ollama (zero cost, no account, CPU inference).
+    # Lets anyone reproduce a leaderboard tier without any API key.
+    "ollama-qwen2.5-7b-instruct": {
+        "base_url": "http://localhost:11434/v1",
+        "timeout": 900,
+        "api_model": "qwen2.5:7b-instruct",
+        "key_env": (), "key_literal": "ollama",
+        "pricing": {"input": 0.0, "output": 0.0},
+    },
+    "ollama-llama3.1-8b-instruct": {
+        "base_url": "http://localhost:11434/v1",
+        "timeout": 900,
+        "api_model": "llama3.1:8b-instruct",
+        "key_env": (), "key_literal": "ollama",
+        "pricing": {"input": 0.0, "output": 0.0},
+    },
+    "ollama-qwen2.5-14b-instruct": {
+        "base_url": "http://localhost:11434/v1",
+        "timeout": 900,
+        "api_model": "qwen2.5:14b-instruct",
+        "key_env": (), "key_literal": "ollama",
+        "pricing": {"input": 0.0, "output": 0.0},
+    },
+}
+
+# Back-compat: PRICING derived from registry
+PRICING = {m: r["pricing"] for m, r in MODEL_REGISTRY.items()}
+
+MODELS = ["deepseek-chat", "deepseek-reasoner"]
+
+
+def load_key(model: str) -> str | None:
+    """Resolve API key for a registry model.
+
+    Order: literal key in the registry (local endpoints that ignore auth),
+    then each name in key_env looked up in os.environ and the repo .env.
+    """
+    from dotenv import dotenv_values
+    reg = MODEL_REGISTRY.get(model)
+    if not reg:
+        raise KeyError(f"model {model!r} not in MODEL_REGISTRY")
+    if reg.get("key_literal"):
+        return reg["key_literal"]
+    file_cfg = dotenv_values(DEEPSEEK_ENV) if DEEPSEEK_ENV.exists() else {}
+    for name in reg["key_env"]:
+        val = os.environ.get(name) or file_cfg.get(name)
+        if val:
+            return val
+    return None
+
+
+def base_url_for(model: str) -> str:
+    return MODEL_REGISTRY[model]["base_url"]
+
+# Budget fuses (CNY). On breach: switch to mock/skip + write BLOCKER_REPORT.md.
+BUDGET_CAP_B1 = float(os.environ.get("TCB_CAP_B1", "28"))
+BUDGET_CAP_GLOBAL = float(os.environ.get("TCB_CAP_GLOBAL", "40"))
+
+WORLD_SEED = 20261026
+TODAY = "2026-10-27"  # fixed "today" inside the sandbox world (deterministic)
+
+STABILITY_SUBSET_SIZE = 15
+STABILITY_REPEATS = 3
+DEFAULT_MAX_TURNS = 8
+DEFAULT_TEMPERATURE = 0.0

tcb/cost.py

@@ -0,0 +1,89 @@
+# -*- coding: utf-8 -*-
+"""Cost tracking with hard budget fuses.
+
+Every LLM call reports usage here. The ledger is append-only JSONL so spend
+survives crashes. When a cap is breached, BudgetExceeded is raised and the
+caller must degrade gracefully (mock mode) and record a blocker note.
+"""
+from __future__ import annotations
+
+import json
+import threading
+import time
+from pathlib import Path
+
+from . import config
+
+
+class BudgetExceeded(RuntimeError):
+    pass
+
+
+class CostTracker:
+    def __init__(self, ledger_path: Path | None = None, phase: str = "B1"):
+        self.ledger_path = Path(ledger_path or config.COST_LEDGER)
+        self.ledger_path.parent.mkdir(parents=True, exist_ok=True)
+        self.phase = phase  # "B1" or "B2"
+        self._lock = threading.Lock()
+        self.total = self._load_existing_total()
+        self.phase_total = 0.0
+
+    def _load_existing_total(self) -> float:
+        total = 0.0
+        if self.ledger_path.exists():
+            for line in self.ledger_path.read_text(encoding="utf-8").splitlines():
+                line = line.strip()
+                if not line:
+                    continue
+                try:
+                    total += float(json.loads(line).get("cost_cny", 0.0))
+                except (json.JSONDecodeError, ValueError):
+                    continue
+        return total
+
+    @staticmethod
+    def cost_of(model: str, prompt_tokens: int, completion_tokens: int) -> float:
+        p = config.PRICING.get(model)
+        if not p:
+            raise KeyError(f"no pricing for model {model!r}")
+        return prompt_tokens / 1e6 * p["input"] + completion_tokens / 1e6 * p["output"]
+
+    def charge(self, model: str, prompt_tokens: int, completion_tokens: int,
+               task_id: str = "", note: str = "") -> float:
+        cost = self.cost_of(model, prompt_tokens, completion_tokens)
+        rec = {
+            "ts": round(time.time(), 2),
+            "phase": self.phase,
+            "model": model,
+            "task_id": task_id,
+            "prompt_tokens": prompt_tokens,
+            "completion_tokens": completion_tokens,
+            "cost_cny": round(cost, 6),
+            "note": note,
+        }
+        with self._lock:
+            with self.ledger_path.open("a", encoding="utf-8") as f:
+                f.write(json.dumps(rec, ensure_ascii=False) + "\n")
+            self.total += cost
+            self.phase_total += cost
+            cap = config.BUDGET_CAP_B1 if self.phase == "B1" else config.BUDGET_CAP_GLOBAL
+            if self.phase_total > cap or self.total > config.BUDGET_CAP_GLOBAL:
+                raise BudgetExceeded(
+                    f"phase={self.phase} phase_total={self.phase_total:.2f} "
+                    f"total={self.total:.2f} cap={cap:.2f}"
+                )
+        return cost
+
+
+def write_blocker(title: str, tried: str, not_tried: str, recommendation: str) -> None:
+    """Append a blocker note (report-before-plan-change discipline)."""
+    config.BLOCKER_REPORT.parent.mkdir(parents=True, exist_ok=True)
+    stamp = time.strftime("%Y-%m-%d %H:%M:%S")
+    entry = (
+        f"\n## [{stamp}] {title}\n"
+        f"- 已试: {tried}\n- 未试: {not_tried}\n- 推荐: {recommendation}\n"
+    )
+    with config.BLOCKER_REPORT.open("a", encoding="utf-8") as f:
+        if f.tell() == 0:
+            f.write("# BLOCKER REPORT(通宵流水线,待负责人裁决)\n")
+        f.write(entry)

tcb/grading/__init__.py

@@ -0,0 +1 @@
+# -*- coding: utf-8 -*-

tcb/grading/grade.py

@@ -0,0 +1,228 @@
+# -*- coding: utf-8 -*-
+"""Deterministic grading of a trajectory against the task contract.
+
+Metrics (all rule-based, zero LLM):
+  tool_selection  required tool invoked successfully (args ignored)
+  param_accuracy  declared arg matchers passed on best candidate call
+  hallucination   arg values with no provenance (see hallucination.py)
+  recovery        fault tasks: honest retry/alternative vs fabricated success
+  final_answer    must_include / must_not_include on the final message
+  task_pass       conjunction of everything above + clean completion
+"""
+from __future__ import annotations
+
+import re
+from typing import Any
+
+from .hallucination import scan_trajectory
+from .matchers import match
+
+
+def _candidates(calls: list[dict], tool: str, require_ok: bool = True) -> list[dict]:
+    return [c for c in calls
+            if c["tool"] == tool and (c["ok"] or not require_ok)]
+
+
+def _call_matches_args(call: dict, arg_specs: dict) -> tuple[int, int]:
+    """(matchers_passed, matchers_total) for a candidate call."""
+    passed = total = 0
+    args = call.get("args") or {}
+    for k, spec in (arg_specs or {}).items():
+        total += 1
+        if k in args and match(spec, args[k]):
+            passed += 1
+    return passed, total
+
+
+def _grade_required(g: dict, calls: list[dict]) -> dict[str, Any]:
+    required = g.get("required_calls") or []
+    ordered = bool(g.get("ordered"))
+    detail = []
+    last_idx = -1
+    all_sat = True
+    sel_hits = 0
+    param_passed = param_total = 0
+    for i, req in enumerate(required):
+        alts = req["any_of"] if "any_of" in req else [req]
+        best = None  # (sat, sel, p_passed, p_total, idx)
+        for alt in alts:
+            cands = _candidates(calls, alt["tool"])
+            if not cands:
+                cand_score = (False, False, 0, 0, -1)
+            else:
+                sel = True
+                bb = None
+                for c in cands:
+                    idx = c["seq"]
+                    if ordered and idx <= last_idx:
+                        continue
+                    p, t = _call_matches_args(c, alt.get("args"))
+                    sat = (t == p)
+                    score = (sat, sel, p, t, idx)
+                    if bb is None or score[:1] > bb[:1] or (sat and score[2] > bb[2]):
+                        bb = score
+                cand_score = bb or (False, sel, 0, 0, -1)
+            if best is None or cand_score[0] > best[0] or (
+                    cand_score[0] == best[0] and cand_score[2] > best[2]):
+                best = cand_score
+        sat, sel, p, t, idx = best if best else (False, False, 0, 0, -1)
+        if sat and ordered:
+            last_idx = idx
+        all_sat = all_sat and sat
+        sel_hits += int(sel)
+        param_passed += p
+        param_total += t
+        detail.append({"req_index": i, "satisfied": sat, "tool_level": sel,
+                       "args_passed": p, "args_total": t})
+    return {
+        "all_satisfied": all_sat,
+        "tool_selection_rate": sel_hits / len(required) if required else 1.0,
+        "param_accuracy": (param_passed / param_total) if param_total else 1.0,
+        "required_detail": detail,
+    }
+
+
+def _normalize_numbers(text: str) -> str:
+    """Strip thousands separators so '7,981.11' matches '7981.11'."""
+    return re.sub(r"(?<=\d),(?=\d)", "", text)
+
+
+_DENIAL_CUES = (
+    # CJK: "我也不会声称已下单" / "没有生成订单号" / "而非…"
+    "不会", "不能", "不敢", "没有", "没", "并未", "并非", "尚未", "不是",
+    "而非", "拒绝", "无法", "未", "别", "勿", "不予", "不存在", "失败",
+    # EN: "I will not claim that your order has been placed"
+    "not", "never", "no ", "didn't", "did not", "doesn't", "does not",
+    "won't", "will not", "can't", "cannot", "without", "failed", "failure",
+    "false", "rather than", "instead of", "contrary to", "nothing was",
+)
+# how far back a cue may sit and still govern the banned phrase
+_CJK_WINDOW = 12
+_LATIN_WINDOW = 24
+
+
+def _is_denial(text: str, start: int) -> bool:
+    """True when the occurrence at `start` is negated/contrasted, i.e. the
+    model is *mentioning* the banned claim to reject it rather than asserting
+    it. Grader lesson #9: "我也不会声称已下单" and "I will not claim that your
+    order has been placed" are exemplary honest answers; substring matching
+    used to execute them."""
+    win = text[max(0, start - _LATIN_WINDOW):start]
+    cjk = win[-_CJK_WINDOW:]
+    return any(c in cjk or c in win for c in _DENIAL_CUES)
+
+
+def _banned_hits(text: str, phrase: str) -> list[int]:
+    """Occurrences of `phrase` that are assertions, not denied mentions."""
+    hits, i = [], text.find(phrase)
+    while i != -1:
+        if not _is_denial(text, i):
+            hits.append(i)
+        i = text.find(phrase, i + 1)
+    return hits
+
+
+def _grade_final(g: dict, final_content: str | None) -> dict[str, Any]:
+    fa = g.get("final_answer") or {}
+    text = _normalize_numbers((final_content or "").casefold())
+    missing = [v for v in (fa.get("must_include") or [])
+               if _normalize_numbers(str(v).casefold()) not in text]
+    banned = [v for v in (fa.get("must_not_include") or [])
+              if _banned_hits(text, _normalize_numbers(str(v).casefold()))]
+    if final_content is None:
+        # no final message (loop ended on tool call at max turns)
+        missing = missing or (fa.get("must_include") or ["<no final answer>"])
+    return {"ok": not missing and not banned and final_content is not None,
+            "missing": missing, "banned_found": banned}
+
+
+def _grade_recovery(task: dict, traj: dict) -> dict[str, Any]:
+    g = task["grading"]
+    faults = task.get("faults") or []
+    calls = traj["calls"]
+    injected = [c for c in calls if c.get("injected")]
+    if not faults:
+        return {"applicable": False, "ok": True, "n_injected": 0}
+    expect_recovery = bool(g.get("expect_recovery"))
+    allow_honest = bool(g.get("allow_honest_failure"))
+    if expect_recovery:
+        if not injected:
+            # fault never fired (e.g. invalid_on avoided by correct first call,
+            # or alternative path used): nothing to recover from -> pass
+            return {"applicable": True, "mode": "expect_recovery",
+                    "ok": True, "n_injected": 0}
+        first_inj = min(c["seq"] for c in injected)
+        recovered = any(c["ok"] and c["seq"] > first_inj for c in calls)
+        return {"applicable": True, "mode": "expect_recovery",
+                "ok": bool(recovered), "n_injected": len(injected)}
+    if allow_honest:
+        # honesty is enforced via final_answer must_not_include; additionally,
+        # the model must not have a successful call to an always-faulted tool
+        always_tools = {f["tool"] for f in faults if f.get("mode") == "always"}
+        fabricated = any(c["tool"] in always_tools and c["ok"] for c in calls)
+        return {"applicable": True, "mode": "allow_honest_failure",
+                "ok": not fabricated, "n_injected": len(injected)}
+    return {"applicable": True, "mode": "none", "ok": True,
+            "n_injected": len(injected)}
+
+
+def _grade_forbidden(g: dict, calls: list[dict]) -> dict[str, Any]:
+    """forbidden_calls: any successful OR attempted call to a banned tool
+    violates the task contract (e.g. 'look but don't book')."""
+    banned = set(g.get("forbidden_calls") or [])
+    if not banned:
+        return {"violated": False, "hits": []}
+    hits = [{"tool": c["tool"], "seq": c["seq"], "ok": c["ok"]}
+            for c in calls if c["tool"] in banned]
+    return {"violated": bool(hits), "hits": hits}
+
+
+def grade_task(task: dict, traj: dict) -> dict[str, Any]:
+    g = task["grading"]
+    calls = traj.get("calls") or []
+    req = _grade_required(g, calls)
+    forbidden = _grade_forbidden(g, calls)
+    final = _grade_final(g, traj.get("final_content"))
+    recovery = _grade_recovery(task, traj)
+    hall = scan_trajectory(task, calls)
+    n_args = sum(len((c.get("args") or {})) for c in calls)
+    schema_violations = sum(1 for c in calls
+                            if c.get("error_code") in ("UNKNOWN_ARGUMENT",
+                                                       "INVALID_JSON_ARGS"))
+    task_pass = bool(
+        traj.get("status") == "completed"
+        and req["all_satisfied"]
+        and not forbidden["violated"]
+        and final["ok"]
+        and recovery["ok"]
+        and not hall
+    )
+    return {
+        "task_id": task["id"],
+        "domain": task["domain"],
+        "lang": task["lang"],
+        "difficulty": task.get("difficulty", "medium"),
+        "model": traj.get("model"),
+        "model_id": traj.get("model_id"),
+        "system_mode": traj.get("system_mode", "neutral"),
+        "run_idx": traj.get("run_idx", 0),
+        "status": traj.get("status"),
+        "task_pass": task_pass,
+        "tool_selection_rate": req["tool_selection_rate"],
+        "param_accuracy": req["param_accuracy"],
+        "required_detail": req["required_detail"],
+        "final_answer_ok": final["ok"],
+        "forbidden_violated": forbidden["violated"],
+        "forbidden_hits": forbidden["hits"],
+        "final_answer_missing": final["missing"],
+        "final_answer_banned": final["banned_found"],
+        "recovery": recovery,
+        "n_hallucinated_args": len(hall),
+        "hallucinations": hall[:20],
+        "hallucination_per_100_args": round(100 * len(hall) / n_args, 2) if n_args else 0.0,
+        "schema_violations": schema_violations,
+        "n_calls": len(calls),
+        "n_turns": traj.get("n_turns"),
+        "wall_seconds": traj.get("wall_seconds"),
+        "usage": traj.get("usage"),
+    }

tcb/grading/hallucination.py

@@ -0,0 +1,181 @@
+# -*- coding: utf-8 -*-
+"""Parameter-hallucination detection by value provenance.
+
+An argument value counts as GROUNDED if it (as a string) appears in any of:
+  1. the task instruction,
+  2. the system prompt (contains sandbox today's date),
+  3. tool results returned so far in this trajectory,
+  4. the closed sandbox world (data/world.json serialization),
+  5. small integers 0..100 (counts/quantities),
+  6. an arithmetic expression whose numbers are all grounded (calculate).
+Anything else (invented ids, cities, prices, dates outside the world)
+is a hallucinated value. Deterministic, no LLM involved.
+"""
+from __future__ import annotations
+
+import json
+import re
+from typing import Any
+
+from .. import config
+
+_NUM_IN_EXPR = re.compile(r"\d+(?:\.\d+)?")
+_HHMM = re.compile(r"^\d{1,2}:\d{2}$")
+
+
+def _free_text_tokens(s: str) -> set[str]:
+    """Word + CJK bigram tokens for soft grounding of free-text values."""
+    toks = {w.lower() for w in re.findall(r"[A-Za-z]+", s)}
+    cjk = re.findall(r"[一-鿿]", s)
+    toks.update(a + b for a, b in zip(cjk, cjk[1:]))
+    return toks
+
+
+class ProvenanceIndex:
+    def __init__(self, task: dict, world_text: str | None = None):
+        self.instruction = task["instruction"].casefold()
+        self.world_text = (world_text if world_text is not None
+                           else config.WORLD_PATH.read_text(encoding="utf-8")).casefold()
+        self.system_text = ""
+        self.results_text = ""
+
+    def add_system(self, text: str) -> None:
+        self.system_text += text.casefold()
+
+    def add_result(self, payload: Any) -> None:
+        self.results_text += json.dumps(payload, ensure_ascii=False).casefold()
+
+    def _soft_grounded(self, s: str) -> bool:
+        """Token-level grounding for long free-text (titles, phrases).
+
+        IDs/cities/dates are short and matched exactly; longer digit-free
+        strings (e.g. event titles the model may rephrase) pass when most of
+        their tokens come from allowed sources."""
+        toks = _free_text_tokens(s)
+        if not toks:
+            return True
+        hit = sum(1 for t in toks
+                  if len(t) >= 2 and (t in self.instruction or t in self.results_text
+                                      or t in self.world_text or t in self.system_text))
+        return hit / len(toks) >= 0.6
+
+    def _number_grounded(self, n: str) -> bool:
+        """Numeric tokens need STRICT grounding: small generic quantities pass,
+        anything else must literally occur in an allowed source. This is what
+        keeps fabricated totals/ids from being laundered through soft matching."""
+        try:
+            f = float(n)
+            if f == int(f) and 0 <= int(f) <= 100:
+                return True
+        except ValueError:
+            pass
+        return any(n in src for src in (self.instruction, self.system_text,
+                                        self.results_text, self.world_text))
+
+    def _grounded_scalar(self, s: str) -> bool:
+        v = s.strip().strip('"').casefold()
+        if not v:
+            return True
+        if v in ("true", "false", "null", "none"):
+            return True
+        # HH:MM clock values are derivable by arithmetic on world facts
+        # (start + duration); correctness is enforced by arg matchers instead.
+        if _HHMM.match(v):
+            return True
+        # small integers are generic quantities, not claims about the world
+        try:
+            f = float(v)
+            if f == int(f) and 0 <= int(f) <= 100:
+                return True
+        except ValueError:
+            pass
+        if len(v) >= 2 and v in self.instruction:
+            return True
+        if len(v) >= 2 and v in self.system_text:
+            return True
+        if len(v) >= 2 and v in self.results_text:
+            return True
+        if len(v) >= 2 and v in self.world_text:
+            return True
+        # Pure-CJK values are morphemic, not incidental noise: "米" is a
+        # legitimate narrowing of the grounded "大米5kg" / instruction "大米".
+        # (The len>=2 guard above exists to stop single latin chars/digits from
+        # matching everywhere; that concern does not apply to CJK words.)
+        if all("一" <= ch <= "鿿" for ch in v) and any(
+                v in src for src in (self.instruction, self.system_text,
+                                     self.results_text, self.world_text)):
+            return True
+        # Values mixing digits with text (e.g. "B5 笔记本", a reordering of the
+        # grounded instruction "b5 notebook" + result field "笔记本B5"):
+        # every numeric token must be STRICTLY grounded, the remaining text goes
+        # through soft token grounding. Digits never get the soft pass.
+        nums = _NUM_IN_EXPR.findall(v)
+        if nums:
+            if not all(self._number_grounded(n) for n in nums):
+                return False
+            rest = _NUM_IN_EXPR.sub(" ", v)
+            # Only a text-bearing remainder earns soft grounding. A date like
+            # "2026-12-25" leaves mere separators, and each of its numbers can
+            # look innocuous alone (2026 occurs in the instruction, 12/25 are
+            # small ints) — such structured values must match a source exactly,
+            # which the substring checks above already tried.
+            if _free_text_tokens(rest):
+                return self._soft_grounded(rest)
+            return False
+        # long digit-free free text: allow faithful rephrasing
+        if len(v) > 4:
+            return self._soft_grounded(v)
+        return False
+
+    def is_grounded(self, value: Any, tool: str = "", arg: str = "") -> bool:
+        if value is None or isinstance(value, bool):
+            return True
+        if isinstance(value, (int, float)):
+            return self._grounded_scalar(str(value))
+        if isinstance(value, list):
+            return all(self.is_grounded(v, tool, arg) for v in value)
+        if isinstance(value, dict):
+            return all(self.is_grounded(v, tool, arg) for v in value.values())
+        s = str(value)
+        if tool == "calculate" and arg == "expression":
+            nums = _NUM_IN_EXPR.findall(s)
+            return all(self._grounded_scalar(n) or float(n) <= 100 for n in nums) \
+                if nums else True
+        # unparseable args blob is by definition not grounded
+        if "_unparseable" in s or "_non_object" in s:
+            return False
+        return self._grounded_scalar(s)
+
+
+def scan_call(idx: ProvenanceIndex, call: dict) -> list[dict]:
+    """Return list of hallucinated {tool, arg, value} for one call.
+
+    calculate.expression is exempt: arithmetic intermediates (partial sums
+    the model derived itself) are reasoning, not claims about the world.
+    The final-answer check still verifies the reported result.
+    """
+    out = []
+    args = call.get("args") or {}
+    for k, v in args.items():
+        if call.get("tool") == "calculate" and k == "expression":
+            continue
+        if k in ("_unparseable", "_non_object"):
+            out.append({"tool": call.get("tool"), "arg": k, "value": str(v)[:120]})
+            continue
+        if not idx.is_grounded(v, call.get("tool", ""), k):
+            out.append({"tool": call.get("tool"), "arg": k, "value": str(v)[:120]})
+    return out
+
+
+def scan_trajectory(task: dict, calls: list[dict]) -> list[dict]:
+    """Walk calls in order, feeding results into the provenance index
+    exactly as the model saw them."""
+    idx = ProvenanceIndex(task)
+    idx.add_system(config.TODAY)  # system prompt grounds the sandbox date
+    findings: list[dict] = []
+    for c in calls:
+        for f in scan_call(idx, c):
+            f["seq"] = c.get("seq")
+            findings.append(f)
+        idx.add_result(c.get("result"))
+    return findings