hidevs-eval 0.1.1__tar.gz

hidevs_eval-0.1.1/PKG-INFO

@@ -0,0 +1,6 @@
+Metadata-Version: 2.4
+Name: hidevs-eval
+Version: 0.1.1
+Summary: HiDevs quest eval client. Runs the local agent and uploads replies for scoring.
+Requires-Python: >=3.11
+Requires-Dist: pyyaml>=6

hidevs_eval-0.1.1/README.md

@@ -0,0 +1,115 @@
+# hidevs-eval
+
+Laptop client for the HiDevs quest evaluation. Python 3.11 or newer is required.
+
+This package runs your agent and uploads the replies. The server at `https://cli.server.hidevs.xyz` sends the inputs, scores the replies, and stores the run. This repo does not contain the server.
+
+## Install
+
+From PyPI (after this package is published):
+
+```bash
+pip install -U hidevs-eval
+hidevs-eval version
+```
+
+`hidevs-eval version` should print `0.1.1`.
+
+Until PyPI is live, or if you need the git copy:
+
+```bash
+pip install -U "git+https://github.com/hidevscommunity/hidevs-eval-cli.git"
+```
+
+The git install needs access to that repository.
+
+## User flow
+
+1. Install the client and confirm it with `hidevs-eval version`.
+2. Take the platform token from the quest page. After you select a problem, the token looks like `HQ-1A2B3C4D`. That token is the login code. It names you, the quest, and the problem statement.
+3. Copy the sample file to `agent_config.yaml` in the root of your agent repository. Fill in how your program starts, which field receives each problem input, and which field in the reply holds each problem output. Leave a field out when the program has nowhere to read or write it.
+4. From that same folder, log in and paste the token:
+
+```bash
+hidevs-eval login
+```
+
+The login is saved in `~/.hidevs_quest/session.json`. The CLI prints the user, the quest, and the problem statement.
+
+5. Optional: check that this Python can import the entry and tools from `agent_config.yaml` (not a scored check):
+
+```bash
+hidevs-eval preflight --config agent_config.yaml
+```
+
+`eval` runs the same check automatically before Level 1.
+
+6. Run all three levels:
+
+```bash
+hidevs-eval eval --config agent_config.yaml
+```
+
+Running again stores another result. Earlier results stay.
+
+## Commands
+
+```bash
+hidevs-eval help
+hidevs-eval version
+hidevs-eval login
+hidevs-eval whoami
+hidevs-eval preflight --config agent_config.yaml
+hidevs-eval eval --config agent_config.yaml
+hidevs-eval logout
+```
+
+`--level 1`, `--level 2`, or `--level 3` runs only that level.
+
+```bash
+hidevs-eval server show
+hidevs-eval server set https://cli.server.hidevs.xyz
+```
+
+`server set` clears the saved login. Run `login` again after changing the link.
+
+## What is evaluated
+
+Level 1 runs on all five cases for your problem statement.
+
+- L1.1 the file matches the schema and names the required fields
+- L1.2 the declared entry returned
+- L1.3 the shared input fits the declared input schema
+- L1.4 the declared output paths exist
+- L1.5 the declared normalizers match the expected values
+- L1.6 the shared expected values match
+- L1.7 forbidden values from the input are absent from the reply
+- L1.8 the response matches the declared schema and the problem limits
+- L1.9 the required tools were called
+- L1.10 the required framework is declared and present in the code
+
+Level 2 uses the first two of those cases.
+
+- L2.1 wrong type on the first case. The run must report the failure status, and the required tools must not be called
+- L2.2 wrong type on the second case
+- L2.3 empty value on the first case
+- L2.4 empty value on the second case
+- L2.5 every required tool raises for the whole first case. The run must report the failure status
+- L2.6 every required tool raises for the whole second case
+- L2.7 every required tool times out for the whole first case. The run must report the failure status
+- L2.8 every required tool times out for the whole second case
+- L2.9 the required tools run on the first case
+- L2.10 the required tools run on the second case
+
+Level 3 uses the same first two cases. The server scores each reply.
+
+- L3.1 hallucination on the first case
+- L3.2 hallucination on the second case
+- L3.3 answer relevancy on the first case
+- L3.4 answer relevancy on the second case
+- L3.5 problem rubric on the first case
+- L3.6 problem rubric on the second case
+- L3.7 PII on the first case
+- L3.8 PII on the second case
+- L3.9 bias on the first case
+- L3.10 bias on the second case

hidevs_eval-0.1.1/pyproject.toml

@@ -0,0 +1,18 @@
+[build-system]
+requires = ["setuptools>=68"]
+build-backend = "setuptools.build_meta"
+
+[project]
+name = "hidevs-eval"
+version = "0.1.1"
+description = "HiDevs quest eval client. Runs the local agent and uploads replies for scoring."
+requires-python = ">=3.11"
+dependencies = [
+  "pyyaml>=6",
+]
+
+[project.scripts]
+hidevs-eval = "hidevs_quest.cli:main"
+
+[tool.setuptools.packages.find]
+where = ["src"]

hidevs_eval-0.1.1/setup.cfg

@@ -0,0 +1,4 @@
+[egg_info]
+tag_build = 
+tag_date = 0
+

hidevs_eval-0.1.1/src/hidevs_eval.egg-info/PKG-INFO

@@ -0,0 +1,6 @@
+Metadata-Version: 2.4
+Name: hidevs-eval
+Version: 0.1.1
+Summary: HiDevs quest eval client. Runs the local agent and uploads replies for scoring.
+Requires-Python: >=3.11
+Requires-Dist: pyyaml>=6

hidevs_eval-0.1.1/src/hidevs_eval.egg-info/SOURCES.txt

@@ -0,0 +1,11 @@
+README.md
+pyproject.toml
+src/hidevs_eval.egg-info/PKG-INFO
+src/hidevs_eval.egg-info/SOURCES.txt
+src/hidevs_eval.egg-info/dependency_links.txt
+src/hidevs_eval.egg-info/entry_points.txt
+src/hidevs_eval.egg-info/requires.txt
+src/hidevs_eval.egg-info/top_level.txt
+src/hidevs_quest/__init__.py
+src/hidevs_quest/cli.py
+src/hidevs_quest/runner.py

hidevs_eval-0.1.1/src/hidevs_eval.egg-info/dependency_links.txt

@@ -0,0 +1 @@
+

hidevs_eval-0.1.1/src/hidevs_eval.egg-info/entry_points.txt

@@ -0,0 +1,2 @@
+[console_scripts]
+hidevs-eval = hidevs_quest.cli:main

hidevs_eval-0.1.1/src/hidevs_eval.egg-info/requires.txt

@@ -0,0 +1 @@
+pyyaml>=6

hidevs_eval-0.1.1/src/hidevs_eval.egg-info/top_level.txt

@@ -0,0 +1 @@
+hidevs_quest

hidevs_eval-0.1.1/src/hidevs_quest/__init__.py

@@ -0,0 +1 @@
+__version__ = "0.1.1"

hidevs_eval-0.1.1/src/hidevs_quest/cli.py

@@ -0,0 +1,332 @@
+"""hidevs-eval login, then hidevs-eval eval. The server sends the cases and stores the score."""
+
+from __future__ import annotations
+
+import argparse
+import json
+import sys
+import urllib.error
+from pathlib import Path
+
+import yaml
+
+from hidevs_quest import __version__
+from hidevs_quest.runner import collect_sources, get_json, post_json, preflight, run_cases
+
+SESSION = Path.home() / ".hidevs_quest" / "session.json"
+
+
+def load_session() -> dict:
+    if not SESSION.exists():
+        return {}
+    return json.loads(SESSION.read_text())
+
+
+def save_session(session: dict) -> None:
+    SESSION.parent.mkdir(parents=True, exist_ok=True)
+    SESSION.write_text(json.dumps(session, indent=2))
+
+
+def server_url(args: argparse.Namespace) -> str:
+    return (args.server or load_session().get("server") or "https://cli.server.hidevs.xyz").rstrip("/")
+
+
+def api(url: str, payload: dict, token: str = "") -> dict:
+    try:
+        return post_json(url, payload, token)
+    except urllib.error.HTTPError as exc:
+        detail = exc.read().decode()
+        try:
+            detail = json.loads(detail).get("detail", detail)
+        except json.JSONDecodeError:
+            pass
+        raise SystemExit(f"{exc.code} {detail}") from exc
+
+
+def version_key(value: str) -> tuple[int, ...]:
+    parts = []
+    for part in value.split("."):
+        digits = "".join(character for character in part if character.isdigit())
+        parts.append(int(digits or "0"))
+    return tuple(parts)
+
+
+def client_notice(base: str) -> dict:
+    try:
+        settings = get_json(f"{base}/v1/settings")
+    except Exception as exc:
+        raise SystemExit(f"could not reach {base}: {exc}") from exc
+    if version_key(__version__) < version_key(settings.get("min_version") or "0.0.0"):
+        extra = f" {settings['message']}" if settings.get("message") else ""
+        raise SystemExit(
+            f"this install is {__version__}. The server requires {settings['min_version']}.{extra}"
+        )
+    if version_key(settings.get("latest_version") or "0.0.0") > version_key(__version__):
+        print(f"a newer package is published: {settings['latest_version']}. You have {__version__}.")
+        if settings.get("message"):
+            print(settings["message"])
+    published = (settings.get("server_url") or "").rstrip("/")
+    if published and published != base:
+        print(f"the published server link is {published}")
+        print(f"switch with: hidevs-eval server set {published}")
+    return settings
+
+
+def login(args: argparse.Namespace) -> dict:
+    previous = load_session()
+    code = args.code or input("Platform code: ").strip()
+    if not code:
+        raise SystemExit("a platform code is required")
+    base = server_url(args)
+    client_notice(base)
+    body = api(f"{base}/v1/auth", {"code": code})
+    if previous.get("user_id") and previous.get("user_id") != body["user_id"]:
+        print(f"replaced login for {previous['user_id']}")
+    session = {
+        "server": base,
+        "token": body["token"],
+        "user_id": body["user_id"],
+        "quest_id": body.get("quest_id") or "",
+        "ps_id": body["ps_id"],
+    }
+    save_session(session)
+    print(f"validated for user {body['user_id']}  quest {session['quest_id']}  problem {body['ps_id']}")
+    return session
+
+
+def logout(_: argparse.Namespace) -> None:
+    if SESSION.exists():
+        SESSION.unlink()
+    print("logged out")
+
+
+def whoami(args: argparse.Namespace) -> None:
+    session = load_session()
+    print(f"package {__version__}")
+    base = server_url(args)
+    print(f"server {base}")
+    if not session.get("token"):
+        print("not logged in")
+        return
+    print(f"user {session.get('user_id')}  quest {session.get('quest_id')}  problem {session.get('ps_id')}")
+    client_notice(base)
+
+
+def show_server(args: argparse.Namespace) -> None:
+    print(server_url(args))
+
+
+def set_server(args: argparse.Namespace) -> None:
+    url = args.url.rstrip("/")
+    session = load_session()
+    session["server"] = url
+    session.pop("token", None)
+    session.pop("user_id", None)
+    session.pop("quest_id", None)
+    session.pop("ps_id", None)
+    save_session(session)
+    print(f"server set to {url}")
+    print("log in again with the platform code")
+
+
+HELP = """
+hidevs-eval login              ask for the platform code and save it
+hidevs-eval login --code CODE  save a different code, replacing the previous login
+hidevs-eval logout             forget the saved login
+hidevs-eval whoami             show the package version, server, and user
+hidevs-eval preflight          import entry and tools from agent_config.yaml
+hidevs-eval eval               run level 1, level 2, and level 3
+hidevs-eval eval --level 1 --config agent_config.yaml
+hidevs-eval server show        print the server this laptop uses
+hidevs-eval server set URL     point this laptop at a different server
+hidevs-eval version            print the installed package version
+hidevs-eval help               show this list
+""".strip()
+
+
+def show_help(_: argparse.Namespace) -> None:
+    print(HELP)
+
+
+def show_version(_: argparse.Namespace) -> None:
+    print(__version__)
+
+
+ARENA_URL = "https://app.hidevs.xyz/ai-quest/challenges/ps-lyzr-01/arena"
+LEVEL_HEADINGS = {
+    1: "LEVEL 1: DETERMINISTIC PROPERTY ASSERTIONS",
+    2: "LEVEL 2: INPUT AND TOOL QUALITY",
+    3: "LEVEL 3: LLM JUDGE",
+}
+
+
+def check_score(check: dict) -> float:
+    evidence = check.get("evidence")
+    raw = evidence.get("score") if isinstance(evidence, dict) else None
+    if isinstance(raw, (int, float)):
+        return max(0.0, min(1.0, float(raw)))
+    return 1.0 if check.get("status") == "PASS" else 0.0
+
+
+def print_result(results: list[dict]) -> None:
+    line = "=" * 80
+    first = results[0] if results else {}
+    name = first.get("problem_name") or first.get("ps_id") or "Quest evaluation"
+    summary_text = first.get("problem_summary") or ""
+    print(line)
+    print("BENCHMARK: HiDevs AI Quest Evaluation")
+    print(f"PROBLEM STATEMENT: {name}")
+    if summary_text:
+        print(summary_text)
+    print(line)
+    rows = []
+    passed_all = 0
+    total_all = 0
+    for result in results:
+        print()
+        level = result.get("level")
+        print(f"[{LEVEL_HEADINGS.get(level, f'LEVEL {level}')}]")
+        scores = []
+        passed = 0
+        for check in result.get("checks") or []:
+            score = check_score(check)
+            scores.append(score)
+            status = check.get("status") or "FAIL"
+            if status == "PASS":
+                passed += 1
+            mark = "PASS" if status == "PASS" else status
+            evidence = check.get("evidence") if isinstance(check.get("evidence"), dict) else {}
+            reason = (check.get("reason") or "").split("\n")[0]
+            if level == 3:
+                title = f"{evidence.get('metric', '')} ({evidence.get('case', '')})".strip()
+                print(f"  [{mark}] {check.get('id', ''):<24} | Score: {score:.2f} | {title}")
+                if reason:
+                    print(f"         * Reason: {reason}")
+            else:
+                print(f"  [{mark}] {check.get('id', ''):<24} | Score: {score:.2f} | {reason}")
+        total = len(scores)
+        level_score = (sum(scores) / total * 100.0) if total else 0.0
+        rows.append((LEVEL_HEADINGS.get(level, f"Level {level}"), passed, total, level_score))
+        passed_all += passed
+        total_all += total
+    composite = (sum(row[3] for row in rows) / len(rows)) if rows else 0.0
+    print()
+    print(line)
+    print("FINAL BENCHMARK SCORECARD")
+    print(line)
+    for heading, passed, total, level_score in rows:
+        print(f"{heading:<42}: Passed {passed}/{total:<2} | Level Score: {level_score:.1f}%")
+    print("-" * 80)
+    print(f"TOTAL SYSTEM AGENT COMPOSITE SCORE : {composite:.1f}% ({passed_all}/{total_all} Passed)")
+    print(line)
+    print()
+    print(f"For the detailed report, open {ARENA_URL}")
+
+
+def run_preflight(config_path: Path) -> dict:
+    if not config_path.exists():
+        raise SystemExit(f"missing {config_path}")
+    config = yaml.safe_load(config_path.read_text()) or {}
+    root = config_path.parent
+    errors = preflight(config, root)
+    if errors:
+        print("preflight failed (not a scored check)")
+        for item in errors:
+            print(f"  - {item}")
+        raise SystemExit(1)
+    entry = config.get("entry") or {}
+    if entry.get("type") == "http":
+        label = f"http {entry.get('app')}"
+    elif "class" in entry:
+        label = f"python class {entry.get('class')}.{entry.get('method')}"
+    else:
+        label = f"python callable {entry.get('callable')}"
+    print(f"preflight ok: {label}")
+    tools = config.get("tools") or []
+    if tools:
+        print(f"preflight ok: {len(tools)} tool function(s) importable")
+    return config
+
+
+def preflight_command(args: argparse.Namespace) -> None:
+    run_preflight(Path(args.config).resolve())
+
+
+def evaluate(args: argparse.Namespace) -> None:
+    session = load_session()
+    if not session.get("token") or server_url(args) != session.get("server"):
+        session = login(args)
+    else:
+        client_notice(session["server"])
+    config_path = Path(args.config).resolve()
+    config = run_preflight(config_path)
+    root = config_path.parent
+    levels = [args.level] if args.level else [1, 2, 3]
+    reports = []
+    sources = collect_sources(root)
+    for level in levels:
+        started = api(
+            f"{session['server']}/v1/levels/{level}/start",
+            {"config": config, "sources": sources},
+            session["token"],
+        )
+        if started.get("result"):
+            reports.append(started["result"])
+            continue
+        results = run_cases(config, root, started["cases"])
+        reports.append(api(
+            f"{session['server']}/v1/runs/{started['run_id']}",
+            {"results": results},
+            session["token"],
+        ))
+    print_result(reports)
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser(prog="hidevs-eval", description=HELP, formatter_class=argparse.RawDescriptionHelpFormatter)
+    parser.add_argument("--server", default="", help="eval server for this command")
+    sub = parser.add_subparsers(dest="command", required=True)
+
+    help_parser = sub.add_parser("help", help="list the commands")
+    help_parser.set_defaults(func=show_help)
+
+    version_parser = sub.add_parser("version", help="print the installed package version")
+    version_parser.set_defaults(func=show_version)
+
+    login_parser = sub.add_parser("login", help="save a platform code, replacing any previous login")
+    login_parser.add_argument("--code", default="")
+    login_parser.set_defaults(func=login)
+
+    logout_parser = sub.add_parser("logout", help="forget the saved login")
+    logout_parser.set_defaults(func=logout)
+
+    whoami_parser = sub.add_parser("whoami", help="show version, server, and user")
+    whoami_parser.set_defaults(func=whoami)
+
+    server_parser = sub.add_parser("server", help="show or change the server link")
+    server_sub = server_parser.add_subparsers(dest="server_command", required=True)
+    server_show = server_sub.add_parser("show", help="print the server this laptop uses")
+    server_show.set_defaults(func=show_server)
+    server_set = server_sub.add_parser("set", help="point this laptop at a different server")
+    server_set.add_argument("url")
+    server_set.set_defaults(func=set_server)
+
+    preflight_parser = sub.add_parser("preflight", help="import entry and tools before scoring")
+    preflight_parser.add_argument("--config", default="agent_config.yaml")
+    preflight_parser.set_defaults(func=preflight_command)
+
+    eval_parser = sub.add_parser("eval", help="pull cases, run the local agent, upload replies")
+    eval_parser.add_argument("--config", default="agent_config.yaml")
+    eval_parser.add_argument("--level", type=int, default=None)
+    eval_parser.add_argument("--code", default="")
+    eval_parser.set_defaults(func=evaluate)
+
+    args = parser.parse_args()
+    try:
+        args.func(args)
+    except KeyboardInterrupt:
+        sys.exit(130)
+
+
+if __name__ == "__main__":
+    main()

hidevs_eval-0.1.1/src/hidevs_quest/runner.py

@@ -0,0 +1,243 @@
+"""Call the local agent. The server never sees this process."""
+
+from __future__ import annotations
+
+import importlib
+import json
+import sys
+import traceback
+import urllib.request
+from pathlib import Path
+from typing import Any
+
+
+SKIP_DIRS = {".venv", "venv", ".git", "__pycache__", "node_modules", ".hidevs"}
+MAX_FILE_BYTES = 200_000
+MAX_TOTAL_BYTES = 1_500_000
+
+
+def as_dict(value: Any) -> dict:
+    if hasattr(value, "model_dump"):
+        return value.model_dump()
+    if isinstance(value, dict):
+        return value
+    return {"_raw": str(value)}
+
+
+def collect_sources(root: Path) -> dict[str, str]:
+    sources: dict[str, str] = {}
+    total = 0
+    for path in root.rglob("*.py"):
+        if any(part in SKIP_DIRS for part in path.parts):
+            continue
+        if path.stat().st_size > MAX_FILE_BYTES:
+            continue
+        text = path.read_text(errors="ignore")
+        total += len(text)
+        if total > MAX_TOTAL_BYTES:
+            break
+        sources[path.relative_to(root).as_posix()] = text
+    return sources
+
+
+def build_kwargs(config: dict, case: dict) -> dict:
+    entry = config["entry"]
+    kwargs = dict(entry.get("constants") or {})
+    for source, target in (entry.get("arguments") or {}).items():
+        if source in case["input"]:
+            kwargs[target] = case["input"][source]
+    return kwargs
+
+
+def _load_symbol(root: Path, spec: str):
+    if ":" not in spec:
+        raise ValueError(f"expected module:name, got {spec!r}")
+    module_name, attr = spec.split(":", 1)
+    if str(root) not in sys.path:
+        sys.path.insert(0, str(root))
+    module = importlib.import_module(module_name)
+    if not hasattr(module, attr):
+        raise AttributeError(f"{module_name} has no attribute {attr}")
+    return getattr(module, attr)
+
+
+def preflight(config: dict, root: Path) -> list[str]:
+    """Import the entry and tools from agent_config.yaml. Not a scored check."""
+    errors: list[str] = []
+    root = root.resolve()
+    entry = config.get("entry") or {}
+    try:
+        if entry.get("type") == "http":
+            app = entry.get("app") or ""
+            _load_symbol(root, app)
+            if not entry.get("method") or not entry.get("endpoint"):
+                errors.append("entry.method and entry.endpoint are required for http")
+        elif "class" in entry:
+            cls = _load_symbol(root, entry["class"])
+            method_name = entry.get("method") or ""
+            if not method_name:
+                errors.append("entry.method is required for a python class")
+            elif not callable(getattr(cls, method_name, None)):
+                errors.append(f"class {entry['class']} has no callable method {method_name}")
+        elif "callable" in entry:
+            fn = _load_symbol(root, entry["callable"])
+            if not callable(fn):
+                errors.append(f"entry.callable {entry['callable']} is not callable")
+        else:
+            errors.append("entry must declare http app, python class, or python callable")
+    except Exception as exc:
+        errors.append(f"entry could not be imported: {exc}")
+
+    for tool in config.get("tools") or []:
+        spec = tool.get("function") or ""
+        capability = tool.get("capability") or spec
+        if not spec:
+            errors.append(f"tool {capability} is missing function")
+            continue
+        try:
+            fn = _load_symbol(root, spec)
+            if not callable(fn):
+                errors.append(f"tool {capability} function {spec} is not callable")
+        except Exception as exc:
+            errors.append(f"tool {capability} could not be imported ({spec}): {exc}")
+
+    req = root / "requirements.txt"
+    if req.is_file() and errors:
+        errors.append(
+            f"install agent libraries in this same Python, then retry: "
+            f"pip install -r {req}"
+        )
+    elif errors:
+        errors.append(
+            "install agent libraries in the same Python that runs hidevs-eval, then retry"
+        )
+    return errors
+
+
+def invoke(config: dict, root: Path, case: dict) -> Any:
+    entry = config["entry"]
+    kwargs = build_kwargs(config, case)
+    if entry.get("type") == "http":
+        application = _load_symbol(root, entry["app"])
+        from fastapi.testclient import TestClient
+
+        client = TestClient(application)
+        response = client.request(entry["method"], entry["endpoint"], json=kwargs)
+        response.raise_for_status()
+        return response.json()
+    if "class" in entry:
+        cls = _load_symbol(root, entry["class"])
+        method = getattr(cls(), entry["method"])
+        return method(**kwargs)
+    fn = _load_symbol(root, entry["callable"])
+    return fn(**kwargs)
+
+
+class Scripted:
+    def __init__(self, original, mode: str):
+        self.original = original
+        self.mode = mode
+        self.calls = []
+
+    def __call__(self, *args, **kwargs):
+        self.calls.append(True)
+        if self.mode == "raise":
+            raise RuntimeError("injected tool failure")
+        if self.mode == "timeout":
+            raise TimeoutError("injected tool timeout")
+        return self.original(*args, **kwargs)
+
+
+def modules_under(root: Path):
+    root = root.resolve()
+    for module in list(sys.modules.values()):
+        path = getattr(module, "__file__", None)
+        if not path:
+            continue
+        try:
+            if Path(path).resolve().is_relative_to(root):
+                yield module
+        except (OSError, ValueError):
+            continue
+
+
+def patch_tools(root: Path, config: dict, mode: str):
+    wrappers = {}
+    restores = []
+    mode = mode if mode in {"raise", "timeout"} else "ok"
+    sys.path.insert(0, str(root))
+    for tool in config.get("tools") or []:
+        capability = tool.get("capability")
+        spec = tool.get("function") or ""
+        if not capability or ":" not in spec:
+            continue
+        module_name, fn_name = spec.split(":", 1)
+        module = __import__(module_name, fromlist=[fn_name])
+        original = getattr(module, fn_name)
+        wrapper = Scripted(original, mode)
+        wrappers[capability] = wrapper
+        setattr(module, fn_name, wrapper)
+        restores.append((module, fn_name, original))
+        for loaded in modules_under(root):
+            for attr, value in list(vars(loaded).items()):
+                if value is original:
+                    setattr(loaded, attr, wrapper)
+                    restores.append((loaded, attr, original))
+                if isinstance(value, dict):
+                    for key, item in list(value.items()):
+                        if item is original:
+                            value[key] = wrapper
+                            restores.append((value, key, original))
+    return wrappers, restores
+
+
+def restore(restores) -> None:
+    for owner, key, original in reversed(restores):
+        if isinstance(owner, dict):
+            owner[key] = original
+        else:
+            setattr(owner, key, original)
+
+
+def run_cases(config: dict, root: Path, cases: list[dict]) -> list[dict]:
+    results = []
+    for case in cases:
+        mode = case.get("tool_mode") or "ok"
+        wrappers, restores = patch_tools(root, config, mode)
+        try:
+            try:
+                payload = as_dict(invoke(config, root, case))
+                error = ""
+            except Exception:
+                payload = {}
+                error = traceback.format_exc()
+        finally:
+            restore(restores)
+        counts = {name: len(wrapper.calls) for name, wrapper in wrappers.items()}
+        results.append({
+            "id": case["id"],
+            "payload": payload,
+            "error": error[-1500:] if error else "",
+            "counts": counts,
+        })
+    return results
+
+
+CLIENT_HEADER = "X-Hidevs-Client"
+CLIENT_VALUE = "hidevs-eval"
+
+
+def get_json(url: str) -> dict:
+    request = urllib.request.Request(url, headers={CLIENT_HEADER: CLIENT_VALUE})
+    with urllib.request.urlopen(request) as response:
+        return json.loads(response.read().decode())
+
+
+def post_json(url: str, payload: dict, token: str = "") -> dict:
+    data = json.dumps(payload).encode()
+    headers = {"Content-Type": "application/json", CLIENT_HEADER: CLIENT_VALUE}
+    if token:
+        headers["Authorization"] = f"Bearer {token}"
+    request = urllib.request.Request(url, data=data, headers=headers)
+    with urllib.request.urlopen(request) as response:
+        return json.loads(response.read().decode())