hidevs-eval 0.1.1__py3-none-any.whl

hidevs_eval-0.1.1.dist-info/METADATA

@@ -0,0 +1,6 @@
+Metadata-Version: 2.4
+Name: hidevs-eval
+Version: 0.1.1
+Summary: HiDevs quest eval client. Runs the local agent and uploads replies for scoring.
+Requires-Python: >=3.11
+Requires-Dist: pyyaml>=6

hidevs_eval-0.1.1.dist-info/RECORD

@@ -0,0 +1,8 @@
+hidevs_quest/__init__.py,sha256=rnObPjuBcEStqSO0S6gsdS_ot8ITOQjVj_-P1LUUYpg,22
+hidevs_quest/cli.py,sha256=Qdx9MWcf8A4qo6EGUAHX5cjcXOHUhvbJGZ985m11zt4,12112
+hidevs_quest/runner.py,sha256=gtX1EMSa-ZwdRRVtF8OU3mSBja61HB7bbH_lL92kmrw,8452
+hidevs_eval-0.1.1.dist-info/METADATA,sha256=hcrbQxr7qgK2K5srqheLzGLKx8ldqk7NmGGdjoWgm5w,193
+hidevs_eval-0.1.1.dist-info/WHEEL,sha256=mOdXR9C3wMVov7cSY8PY-XCzo3SmmWCHvGQr5csJoao,91
+hidevs_eval-0.1.1.dist-info/entry_points.txt,sha256=M-eo_dbdAwOxU2WBYFGalT0seo03e7oW7oLIFkFlYz4,54
+hidevs_eval-0.1.1.dist-info/top_level.txt,sha256=jkb_W8hpSCXXCEbwvLOs3mjHry2QlDV6a78mA2VFalc,13
+hidevs_eval-0.1.1.dist-info/RECORD,,

hidevs_eval-0.1.1.dist-info/WHEEL

@@ -0,0 +1,5 @@
+Wheel-Version: 1.0
+Generator: setuptools (84.2.1)
+Root-Is-Purelib: true
+Tag: py3-none-any
+

hidevs_eval-0.1.1.dist-info/entry_points.txt

@@ -0,0 +1,2 @@
+[console_scripts]
+hidevs-eval = hidevs_quest.cli:main

hidevs_eval-0.1.1.dist-info/top_level.txt

@@ -0,0 +1 @@
+hidevs_quest

hidevs_quest/__init__.py

@@ -0,0 +1 @@
+__version__ = "0.1.1"

hidevs_quest/cli.py

@@ -0,0 +1,332 @@
+"""hidevs-eval login, then hidevs-eval eval. The server sends the cases and stores the score."""
+
+from __future__ import annotations
+
+import argparse
+import json
+import sys
+import urllib.error
+from pathlib import Path
+
+import yaml
+
+from hidevs_quest import __version__
+from hidevs_quest.runner import collect_sources, get_json, post_json, preflight, run_cases
+
+SESSION = Path.home() / ".hidevs_quest" / "session.json"
+
+
+def load_session() -> dict:
+    if not SESSION.exists():
+        return {}
+    return json.loads(SESSION.read_text())
+
+
+def save_session(session: dict) -> None:
+    SESSION.parent.mkdir(parents=True, exist_ok=True)
+    SESSION.write_text(json.dumps(session, indent=2))
+
+
+def server_url(args: argparse.Namespace) -> str:
+    return (args.server or load_session().get("server") or "https://cli.server.hidevs.xyz").rstrip("/")
+
+
+def api(url: str, payload: dict, token: str = "") -> dict:
+    try:
+        return post_json(url, payload, token)
+    except urllib.error.HTTPError as exc:
+        detail = exc.read().decode()
+        try:
+            detail = json.loads(detail).get("detail", detail)
+        except json.JSONDecodeError:
+            pass
+        raise SystemExit(f"{exc.code} {detail}") from exc
+
+
+def version_key(value: str) -> tuple[int, ...]:
+    parts = []
+    for part in value.split("."):
+        digits = "".join(character for character in part if character.isdigit())
+        parts.append(int(digits or "0"))
+    return tuple(parts)
+
+
+def client_notice(base: str) -> dict:
+    try:
+        settings = get_json(f"{base}/v1/settings")
+    except Exception as exc:
+        raise SystemExit(f"could not reach {base}: {exc}") from exc
+    if version_key(__version__) < version_key(settings.get("min_version") or "0.0.0"):
+        extra = f" {settings['message']}" if settings.get("message") else ""
+        raise SystemExit(
+            f"this install is {__version__}. The server requires {settings['min_version']}.{extra}"
+        )
+    if version_key(settings.get("latest_version") or "0.0.0") > version_key(__version__):
+        print(f"a newer package is published: {settings['latest_version']}. You have {__version__}.")
+        if settings.get("message"):
+            print(settings["message"])
+    published = (settings.get("server_url") or "").rstrip("/")
+    if published and published != base:
+        print(f"the published server link is {published}")
+        print(f"switch with: hidevs-eval server set {published}")
+    return settings
+
+
+def login(args: argparse.Namespace) -> dict:
+    previous = load_session()
+    code = args.code or input("Platform code: ").strip()
+    if not code:
+        raise SystemExit("a platform code is required")
+    base = server_url(args)
+    client_notice(base)
+    body = api(f"{base}/v1/auth", {"code": code})
+    if previous.get("user_id") and previous.get("user_id") != body["user_id"]:
+        print(f"replaced login for {previous['user_id']}")
+    session = {
+        "server": base,
+        "token": body["token"],
+        "user_id": body["user_id"],
+        "quest_id": body.get("quest_id") or "",
+        "ps_id": body["ps_id"],
+    }
+    save_session(session)
+    print(f"validated for user {body['user_id']}  quest {session['quest_id']}  problem {body['ps_id']}")
+    return session
+
+
+def logout(_: argparse.Namespace) -> None:
+    if SESSION.exists():
+        SESSION.unlink()
+    print("logged out")
+
+
+def whoami(args: argparse.Namespace) -> None:
+    session = load_session()
+    print(f"package {__version__}")
+    base = server_url(args)
+    print(f"server {base}")
+    if not session.get("token"):
+        print("not logged in")
+        return
+    print(f"user {session.get('user_id')}  quest {session.get('quest_id')}  problem {session.get('ps_id')}")
+    client_notice(base)
+
+
+def show_server(args: argparse.Namespace) -> None:
+    print(server_url(args))
+
+
+def set_server(args: argparse.Namespace) -> None:
+    url = args.url.rstrip("/")
+    session = load_session()
+    session["server"] = url
+    session.pop("token", None)
+    session.pop("user_id", None)
+    session.pop("quest_id", None)
+    session.pop("ps_id", None)
+    save_session(session)
+    print(f"server set to {url}")
+    print("log in again with the platform code")
+
+
+HELP = """
+hidevs-eval login              ask for the platform code and save it
+hidevs-eval login --code CODE  save a different code, replacing the previous login
+hidevs-eval logout             forget the saved login
+hidevs-eval whoami             show the package version, server, and user
+hidevs-eval preflight          import entry and tools from agent_config.yaml
+hidevs-eval eval               run level 1, level 2, and level 3
+hidevs-eval eval --level 1 --config agent_config.yaml
+hidevs-eval server show        print the server this laptop uses
+hidevs-eval server set URL     point this laptop at a different server
+hidevs-eval version            print the installed package version
+hidevs-eval help               show this list
+""".strip()
+
+
+def show_help(_: argparse.Namespace) -> None:
+    print(HELP)
+
+
+def show_version(_: argparse.Namespace) -> None:
+    print(__version__)
+
+
+ARENA_URL = "https://app.hidevs.xyz/ai-quest/challenges/ps-lyzr-01/arena"
+LEVEL_HEADINGS = {
+    1: "LEVEL 1: DETERMINISTIC PROPERTY ASSERTIONS",
+    2: "LEVEL 2: INPUT AND TOOL QUALITY",
+    3: "LEVEL 3: LLM JUDGE",
+}
+
+
+def check_score(check: dict) -> float:
+    evidence = check.get("evidence")
+    raw = evidence.get("score") if isinstance(evidence, dict) else None
+    if isinstance(raw, (int, float)):
+        return max(0.0, min(1.0, float(raw)))
+    return 1.0 if check.get("status") == "PASS" else 0.0
+
+
+def print_result(results: list[dict]) -> None:
+    line = "=" * 80
+    first = results[0] if results else {}
+    name = first.get("problem_name") or first.get("ps_id") or "Quest evaluation"
+    summary_text = first.get("problem_summary") or ""
+    print(line)
+    print("BENCHMARK: HiDevs AI Quest Evaluation")
+    print(f"PROBLEM STATEMENT: {name}")
+    if summary_text:
+        print(summary_text)
+    print(line)
+    rows = []
+    passed_all = 0
+    total_all = 0
+    for result in results:
+        print()
+        level = result.get("level")
+        print(f"[{LEVEL_HEADINGS.get(level, f'LEVEL {level}')}]")
+        scores = []
+        passed = 0
+        for check in result.get("checks") or []:
+            score = check_score(check)
+            scores.append(score)
+            status = check.get("status") or "FAIL"
+            if status == "PASS":
+                passed += 1
+            mark = "PASS" if status == "PASS" else status
+            evidence = check.get("evidence") if isinstance(check.get("evidence"), dict) else {}
+            reason = (check.get("reason") or "").split("\n")[0]
+            if level == 3:
+                title = f"{evidence.get('metric', '')} ({evidence.get('case', '')})".strip()
+                print(f"  [{mark}] {check.get('id', ''):<24} | Score: {score:.2f} | {title}")
+                if reason:
+                    print(f"         * Reason: {reason}")
+            else:
+                print(f"  [{mark}] {check.get('id', ''):<24} | Score: {score:.2f} | {reason}")
+        total = len(scores)
+        level_score = (sum(scores) / total * 100.0) if total else 0.0
+        rows.append((LEVEL_HEADINGS.get(level, f"Level {level}"), passed, total, level_score))
+        passed_all += passed
+        total_all += total
+    composite = (sum(row[3] for row in rows) / len(rows)) if rows else 0.0
+    print()
+    print(line)
+    print("FINAL BENCHMARK SCORECARD")
+    print(line)
+    for heading, passed, total, level_score in rows:
+        print(f"{heading:<42}: Passed {passed}/{total:<2} | Level Score: {level_score:.1f}%")
+    print("-" * 80)
+    print(f"TOTAL SYSTEM AGENT COMPOSITE SCORE : {composite:.1f}% ({passed_all}/{total_all} Passed)")
+    print(line)
+    print()
+    print(f"For the detailed report, open {ARENA_URL}")
+
+
+def run_preflight(config_path: Path) -> dict:
+    if not config_path.exists():
+        raise SystemExit(f"missing {config_path}")
+    config = yaml.safe_load(config_path.read_text()) or {}
+    root = config_path.parent
+    errors = preflight(config, root)
+    if errors:
+        print("preflight failed (not a scored check)")
+        for item in errors:
+            print(f"  - {item}")
+        raise SystemExit(1)
+    entry = config.get("entry") or {}
+    if entry.get("type") == "http":
+        label = f"http {entry.get('app')}"
+    elif "class" in entry:
+        label = f"python class {entry.get('class')}.{entry.get('method')}"
+    else:
+        label = f"python callable {entry.get('callable')}"
+    print(f"preflight ok: {label}")
+    tools = config.get("tools") or []
+    if tools:
+        print(f"preflight ok: {len(tools)} tool function(s) importable")
+    return config
+
+
+def preflight_command(args: argparse.Namespace) -> None:
+    run_preflight(Path(args.config).resolve())
+
+
+def evaluate(args: argparse.Namespace) -> None:
+    session = load_session()
+    if not session.get("token") or server_url(args) != session.get("server"):
+        session = login(args)
+    else:
+        client_notice(session["server"])
+    config_path = Path(args.config).resolve()
+    config = run_preflight(config_path)
+    root = config_path.parent
+    levels = [args.level] if args.level else [1, 2, 3]
+    reports = []
+    sources = collect_sources(root)
+    for level in levels:
+        started = api(
+            f"{session['server']}/v1/levels/{level}/start",
+            {"config": config, "sources": sources},
+            session["token"],
+        )
+        if started.get("result"):
+            reports.append(started["result"])
+            continue
+        results = run_cases(config, root, started["cases"])
+        reports.append(api(
+            f"{session['server']}/v1/runs/{started['run_id']}",
+            {"results": results},
+            session["token"],
+        ))
+    print_result(reports)
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser(prog="hidevs-eval", description=HELP, formatter_class=argparse.RawDescriptionHelpFormatter)
+    parser.add_argument("--server", default="", help="eval server for this command")
+    sub = parser.add_subparsers(dest="command", required=True)
+
+    help_parser = sub.add_parser("help", help="list the commands")
+    help_parser.set_defaults(func=show_help)
+
+    version_parser = sub.add_parser("version", help="print the installed package version")
+    version_parser.set_defaults(func=show_version)
+
+    login_parser = sub.add_parser("login", help="save a platform code, replacing any previous login")
+    login_parser.add_argument("--code", default="")
+    login_parser.set_defaults(func=login)
+
+    logout_parser = sub.add_parser("logout", help="forget the saved login")
+    logout_parser.set_defaults(func=logout)
+
+    whoami_parser = sub.add_parser("whoami", help="show version, server, and user")
+    whoami_parser.set_defaults(func=whoami)
+
+    server_parser = sub.add_parser("server", help="show or change the server link")
+    server_sub = server_parser.add_subparsers(dest="server_command", required=True)
+    server_show = server_sub.add_parser("show", help="print the server this laptop uses")
+    server_show.set_defaults(func=show_server)
+    server_set = server_sub.add_parser("set", help="point this laptop at a different server")
+    server_set.add_argument("url")
+    server_set.set_defaults(func=set_server)
+
+    preflight_parser = sub.add_parser("preflight", help="import entry and tools before scoring")
+    preflight_parser.add_argument("--config", default="agent_config.yaml")
+    preflight_parser.set_defaults(func=preflight_command)
+
+    eval_parser = sub.add_parser("eval", help="pull cases, run the local agent, upload replies")
+    eval_parser.add_argument("--config", default="agent_config.yaml")
+    eval_parser.add_argument("--level", type=int, default=None)
+    eval_parser.add_argument("--code", default="")
+    eval_parser.set_defaults(func=evaluate)
+
+    args = parser.parse_args()
+    try:
+        args.func(args)
+    except KeyboardInterrupt:
+        sys.exit(130)
+
+
+if __name__ == "__main__":
+    main()

hidevs_quest/runner.py

@@ -0,0 +1,243 @@
+"""Call the local agent. The server never sees this process."""
+
+from __future__ import annotations
+
+import importlib
+import json
+import sys
+import traceback
+import urllib.request
+from pathlib import Path
+from typing import Any
+
+
+SKIP_DIRS = {".venv", "venv", ".git", "__pycache__", "node_modules", ".hidevs"}
+MAX_FILE_BYTES = 200_000
+MAX_TOTAL_BYTES = 1_500_000
+
+
+def as_dict(value: Any) -> dict:
+    if hasattr(value, "model_dump"):
+        return value.model_dump()
+    if isinstance(value, dict):
+        return value
+    return {"_raw": str(value)}
+
+
+def collect_sources(root: Path) -> dict[str, str]:
+    sources: dict[str, str] = {}
+    total = 0
+    for path in root.rglob("*.py"):
+        if any(part in SKIP_DIRS for part in path.parts):
+            continue
+        if path.stat().st_size > MAX_FILE_BYTES:
+            continue
+        text = path.read_text(errors="ignore")
+        total += len(text)
+        if total > MAX_TOTAL_BYTES:
+            break
+        sources[path.relative_to(root).as_posix()] = text
+    return sources
+
+
+def build_kwargs(config: dict, case: dict) -> dict:
+    entry = config["entry"]
+    kwargs = dict(entry.get("constants") or {})
+    for source, target in (entry.get("arguments") or {}).items():
+        if source in case["input"]:
+            kwargs[target] = case["input"][source]
+    return kwargs
+
+
+def _load_symbol(root: Path, spec: str):
+    if ":" not in spec:
+        raise ValueError(f"expected module:name, got {spec!r}")
+    module_name, attr = spec.split(":", 1)
+    if str(root) not in sys.path:
+        sys.path.insert(0, str(root))
+    module = importlib.import_module(module_name)
+    if not hasattr(module, attr):
+        raise AttributeError(f"{module_name} has no attribute {attr}")
+    return getattr(module, attr)
+
+
+def preflight(config: dict, root: Path) -> list[str]:
+    """Import the entry and tools from agent_config.yaml. Not a scored check."""
+    errors: list[str] = []
+    root = root.resolve()
+    entry = config.get("entry") or {}
+    try:
+        if entry.get("type") == "http":
+            app = entry.get("app") or ""
+            _load_symbol(root, app)
+            if not entry.get("method") or not entry.get("endpoint"):
+                errors.append("entry.method and entry.endpoint are required for http")
+        elif "class" in entry:
+            cls = _load_symbol(root, entry["class"])
+            method_name = entry.get("method") or ""
+            if not method_name:
+                errors.append("entry.method is required for a python class")
+            elif not callable(getattr(cls, method_name, None)):
+                errors.append(f"class {entry['class']} has no callable method {method_name}")
+        elif "callable" in entry:
+            fn = _load_symbol(root, entry["callable"])
+            if not callable(fn):
+                errors.append(f"entry.callable {entry['callable']} is not callable")
+        else:
+            errors.append("entry must declare http app, python class, or python callable")
+    except Exception as exc:
+        errors.append(f"entry could not be imported: {exc}")
+
+    for tool in config.get("tools") or []:
+        spec = tool.get("function") or ""
+        capability = tool.get("capability") or spec
+        if not spec:
+            errors.append(f"tool {capability} is missing function")
+            continue
+        try:
+            fn = _load_symbol(root, spec)
+            if not callable(fn):
+                errors.append(f"tool {capability} function {spec} is not callable")
+        except Exception as exc:
+            errors.append(f"tool {capability} could not be imported ({spec}): {exc}")
+
+    req = root / "requirements.txt"
+    if req.is_file() and errors:
+        errors.append(
+            f"install agent libraries in this same Python, then retry: "
+            f"pip install -r {req}"
+        )
+    elif errors:
+        errors.append(
+            "install agent libraries in the same Python that runs hidevs-eval, then retry"
+        )
+    return errors
+
+
+def invoke(config: dict, root: Path, case: dict) -> Any:
+    entry = config["entry"]
+    kwargs = build_kwargs(config, case)
+    if entry.get("type") == "http":
+        application = _load_symbol(root, entry["app"])
+        from fastapi.testclient import TestClient
+
+        client = TestClient(application)
+        response = client.request(entry["method"], entry["endpoint"], json=kwargs)
+        response.raise_for_status()
+        return response.json()
+    if "class" in entry:
+        cls = _load_symbol(root, entry["class"])
+        method = getattr(cls(), entry["method"])
+        return method(**kwargs)
+    fn = _load_symbol(root, entry["callable"])
+    return fn(**kwargs)
+
+
+class Scripted:
+    def __init__(self, original, mode: str):
+        self.original = original
+        self.mode = mode
+        self.calls = []
+
+    def __call__(self, *args, **kwargs):
+        self.calls.append(True)
+        if self.mode == "raise":
+            raise RuntimeError("injected tool failure")
+        if self.mode == "timeout":
+            raise TimeoutError("injected tool timeout")
+        return self.original(*args, **kwargs)
+
+
+def modules_under(root: Path):
+    root = root.resolve()
+    for module in list(sys.modules.values()):
+        path = getattr(module, "__file__", None)
+        if not path:
+            continue
+        try:
+            if Path(path).resolve().is_relative_to(root):
+                yield module
+        except (OSError, ValueError):
+            continue
+
+
+def patch_tools(root: Path, config: dict, mode: str):
+    wrappers = {}
+    restores = []
+    mode = mode if mode in {"raise", "timeout"} else "ok"
+    sys.path.insert(0, str(root))
+    for tool in config.get("tools") or []:
+        capability = tool.get("capability")
+        spec = tool.get("function") or ""
+        if not capability or ":" not in spec:
+            continue
+        module_name, fn_name = spec.split(":", 1)
+        module = __import__(module_name, fromlist=[fn_name])
+        original = getattr(module, fn_name)
+        wrapper = Scripted(original, mode)
+        wrappers[capability] = wrapper
+        setattr(module, fn_name, wrapper)
+        restores.append((module, fn_name, original))
+        for loaded in modules_under(root):
+            for attr, value in list(vars(loaded).items()):
+                if value is original:
+                    setattr(loaded, attr, wrapper)
+                    restores.append((loaded, attr, original))
+                if isinstance(value, dict):
+                    for key, item in list(value.items()):
+                        if item is original:
+                            value[key] = wrapper
+                            restores.append((value, key, original))
+    return wrappers, restores
+
+
+def restore(restores) -> None:
+    for owner, key, original in reversed(restores):
+        if isinstance(owner, dict):
+            owner[key] = original
+        else:
+            setattr(owner, key, original)
+
+
+def run_cases(config: dict, root: Path, cases: list[dict]) -> list[dict]:
+    results = []
+    for case in cases:
+        mode = case.get("tool_mode") or "ok"
+        wrappers, restores = patch_tools(root, config, mode)
+        try:
+            try:
+                payload = as_dict(invoke(config, root, case))
+                error = ""
+            except Exception:
+                payload = {}
+                error = traceback.format_exc()
+        finally:
+            restore(restores)
+        counts = {name: len(wrapper.calls) for name, wrapper in wrappers.items()}
+        results.append({
+            "id": case["id"],
+            "payload": payload,
+            "error": error[-1500:] if error else "",
+            "counts": counts,
+        })
+    return results
+
+
+CLIENT_HEADER = "X-Hidevs-Client"
+CLIENT_VALUE = "hidevs-eval"
+
+
+def get_json(url: str) -> dict:
+    request = urllib.request.Request(url, headers={CLIENT_HEADER: CLIENT_VALUE})
+    with urllib.request.urlopen(request) as response:
+        return json.loads(response.read().decode())
+
+
+def post_json(url: str, payload: dict, token: str = "") -> dict:
+    data = json.dumps(payload).encode()
+    headers = {"Content-Type": "application/json", CLIENT_HEADER: CLIENT_VALUE}
+    if token:
+        headers["Authorization"] = f"Bearer {token}"
+    request = urllib.request.Request(url, data=data, headers=headers)
+    with urllib.request.urlopen(request) as response:
+        return json.loads(response.read().decode())