aidong-aeval 0.1.0__py3-none-any.whl

aeval_client/__init__.py

@@ -0,0 +1 @@
+"""开发者评测客户端，不依赖数据库或 Langfuse SDK。"""

aeval_client/agent.py

@@ -0,0 +1,57 @@
+"""main/v1 的 HTTP 文本执行；过程观测交给 Agent 内置 Langfuse。"""
+
+import json
+import httpx
+
+
+def text_content(message):
+    value = (message.get("kwargs") or message).get("content")
+    if isinstance(value, str):
+        return value
+    if isinstance(value, list):
+        return "\n".join(b["text"] for b in value if isinstance(b, dict) and b.get("type") == "text")
+    raise ValueError("最终回答不是文本")
+
+
+class Agent:
+    def __init__(self, target, protocol, user_id, effort=None, token=None):
+        self.target, self.protocol, self.effort = target, protocol, effort
+        headers = {"X-User-Id": user_id}
+        if token:
+            headers["Authorization"] = "Bearer " + token
+        self.client = httpx.Client(headers=headers, timeout=httpx.Timeout(900, connect=10), follow_redirects=False)
+
+    def close(self):
+        self.client.close()
+
+    def validate(self):
+        if self.effort is None:
+            return
+        if self.protocol != "v1":
+            raise ValueError("main 不支持请求级 effort")
+        response = self.client.get(self.target + "/api/agent/model")
+        response.raise_for_status()
+        levels = response.json()["thinking"]["levels"]
+        if self.effort not in levels:
+            raise ValueError(f"不支持思考档位 {self.effort}，可选：{', '.join(levels)}")
+
+    def ask(self, question, thread_id, history=None):
+        body = {"question": question, "configurable": {"thread_id": thread_id}}
+        if history:
+            body["chat_history"] = history
+        if self.effort:
+            body["effort"] = self.effort
+        answer = None
+        with self.client.stream("POST", self.target + "/api/agent/langgraph", json=body) as response:
+            response.raise_for_status()
+            for line in response.iter_lines():
+                if not line.startswith("data: "):
+                    continue
+                event = json.loads(line[6:])
+                if event.get("status") == "error":
+                    raise ValueError("Agent 返回错误事件，请检查 Agent 日志")
+                if event.get("event") == "on_chain_end" and event.get("name") == "LangGraph":
+                    answer = text_content(event["data"]["output"]["messages"][-1])
+        if answer is None:
+            raise ValueError("响应缺少最终回答，调用是否完成需要核对 trace")
+        return answer

aeval_client/cli.py

@@ -0,0 +1,222 @@
+"""开发机评测入口，恢复时以中心服务的固定清单为准。"""
+
+import argparse
+import json
+import threading
+import time
+from uuid import uuid4
+
+import httpx
+
+from aeval_client import config
+from aeval_client.agent import Agent
+
+
+class ApiError(ValueError):
+    def __init__(self, status, detail):
+        self.status = status
+        super().__init__(detail)
+
+
+class Service:
+    def __init__(self, cfg):
+        self.base = cfg["server"]
+        self.client = httpx.Client(headers={"X-Eval-Token": cfg["api_key"]},
+                                   timeout=180, follow_redirects=False)
+
+    def call(self, method, path, **kwargs):
+        response = self.client.request(method, self.base + "/api/client" + path, **kwargs)
+        if not response.is_success:
+            try:
+                detail = response.json().get("detail")
+            except ValueError:
+                detail = None
+            raise ApiError(response.status_code, detail if isinstance(detail, str) else f"评测服务返回 HTTP {response.status_code}")
+        return response.json()
+
+    def close(self):
+        self.client.close()
+
+
+def save(path, value):
+    config.private_write(path, json.dumps(value, ensure_ascii=False))
+
+
+def parser():
+    p = argparse.ArgumentParser(description="在开发机执行 OmniAgent 评测")
+    p.add_argument("command", nargs="?", choices=["configure", "datasets", "status", "results", "resume"])
+    p.add_argument("run_id", nargs="?")
+    p.add_argument("--target")
+    p.add_argument("--dataset")
+    p.add_argument("--customer")
+    p.add_argument("--protocol", choices=["main", "v1"])
+    p.add_argument("--revision")
+    p.add_argument("--model")
+    p.add_argument("--effort")
+    p.add_argument("--environment")
+    p.add_argument("--k", type=int, default=1)
+    p.add_argument("--retry-uncertain", action="store_true", help="恢复失败后显式重问未知任务，可能产生重复调用")
+    return p
+
+
+def summary(service, run_id):
+    result = service.call("GET", f"/runs/{run_id}/report")
+    print(json.dumps({"运行": run_id, "状态": result["run"]["status"],
+                      "结果": result["report"]["total"], "指标": result["metrics"]}, ensure_ascii=False, indent=2))
+    print(f"报告：{service.base}/runs/{run_id}")
+    return result
+
+
+def execute(service, cfg, run_id, retry=False):
+    prefix = f"/runs/{run_id}"
+    state = service.call("GET", prefix)
+    spec = state["spec"]
+    cache = config.HOME / "runs" / run_id
+    agent = Agent(spec["target"], spec["protocol"], state["user_id"], spec["effort"], cfg.get("agent_token"))
+    stopped = threading.Event()
+
+    def heartbeats():
+        while not stopped.wait(15):
+            try:
+                service.call("POST", prefix + "/heartbeat")
+            except (httpx.HTTPError, ValueError):
+                pass  # 主请求报告连接错误；心跳失败不能伪造成功。
+
+    pulse = threading.Thread(target=heartbeats, daemon=True)
+    pulse.start()
+    try:
+        service.call("POST", prefix + "/heartbeat")
+        # 上次回答已写本地但上传未确认时，先原样重传。
+        for pending in sorted(cache.glob("*.json")):
+            service.call("PUT", prefix + f"/tasks/{pending.stem}", json=json.loads(pending.read_text()))
+            pending.unlink()
+        state = service.call("GET", prefix)
+        for task in state["tasks"]:
+            if task["state"] != "issued":
+                continue
+            try:
+                service.call("POST", prefix + f"/tasks/{task['id']}/recover")
+            except ValueError:
+                if not retry:
+                    raise ValueError("存在未知调用，已尝试从 trace 恢复。稍后 resume；确认可重问时加 --retry-uncertain")
+                service.call("POST", prefix + f"/tasks/{task['id']}/retry")
+        state = service.call("GET", prefix)
+        if not state["approved"] or any(task["state"] == "pending" for task in state["tasks"]):
+            agent.validate()
+        if not state["approved"]:
+            marker = cache / "probe"
+            # 探针也可能在响应丢失前执行完成；同一会话不自动再问。
+            if not marker.exists() or marker.read_text() != state["probe_thread"]:
+                config.private_write(marker, state["probe_thread"])
+                agent.ask("你好，请只回复“收到”两个字", state["probe_thread"])
+            result = service.call("POST", prefix + "/preflight")
+            print(f"安全检查通过，观测环境：{result['environment']}", flush=True)
+        tasks = service.call("GET", prefix)["tasks"]
+        for index, task in enumerate(tasks):
+            if task["state"] != "pending":
+                continue
+            q = service.call("POST", prefix + f"/tasks/{task['id']}/claim")
+            start = time.monotonic()
+            # 断流或超时留下 issued；不把未知调用误报为可安全重试的失败。
+            answer = agent.ask(q["question"], q["thread_id"], q.get("chat_history"))
+            payload = {"thread_id": q["thread_id"], "answer": answer, "elapsed_s": time.monotonic() - start}
+            path = cache / f"{task['id']}.json"
+            save(path, payload)
+            service.call("PUT", prefix + f"/tasks/{task['id']}", json=payload)
+            path.unlink()
+            print(f"回答进度：{index + 1}/{len(tasks)}", flush=True)
+        service.call("POST", prefix + "/finish")
+        last = None
+        while True:
+            state = service.call("GET", prefix)
+            done = sum(t["state"] in ("done", "failed") for t in state["tasks"])
+            progress = (done, state["status"])
+            if progress != last:
+                print(f"判定进度：{done}/{len(state['tasks'])} · {state['status']}", flush=True)
+                last = progress
+            if state["status"] not in ("running", "judging"):
+                break
+            time.sleep(3)
+        summary(service, run_id)
+        (cache / "probe").unlink(missing_ok=True)
+        if cache.exists() and not any(cache.iterdir()):
+            cache.rmdir()
+        return 0 if state["status"] == "done" else 2
+    finally:
+        stopped.set()
+        pulse.join(timeout=1)
+        agent.close()
+
+
+def main(argv=None):
+    args = parser().parse_args(argv)
+    service = None
+    run_id = args.run_id
+    try:
+        if args.command == "configure":
+            config.configure()
+            return 0
+        cfg = config.read()
+        service = Service(cfg)
+        if args.command == "datasets":
+            result = service.call("GET", "/datasets", params={"customer": args.customer or cfg["customer"], "refresh": "true"})
+            for item in result["items"]:
+                print(f"{item['id']}\t{item['name']}\t{item['count']} 题")
+            return 0
+        if args.command in ("status", "results", "resume"):
+            if not run_id:
+                raise ValueError("请指定运行编号")
+            if args.command == "status":
+                state = service.call("GET", f"/runs/{run_id}")
+                print(json.dumps({k: state[k] for k in ("run_id", "status", "environment", "disconnected", "tasks")},
+                                 ensure_ascii=False, indent=2))
+                return 0
+            if args.command == "results":
+                summary(service, run_id)
+                return 0
+            if any(getattr(args, k) is not None for k in ("target", "dataset", "customer", "protocol", "revision", "model", "effort", "environment")):
+                raise ValueError("恢复不能更换原运行参数，请新建评测")
+            service.call("POST", f"/runs/{run_id}/resume")
+            return execute(service, cfg, run_id, args.retry_uncertain)
+        if not args.target:
+            raise ValueError("请指定 --target")
+        dataset = args.dataset or cfg.get("dataset")
+        if not dataset:
+            raise ValueError("请先运行 aeval datasets，再用 --dataset 选择 Langfuse 数据集")
+        spec = {"customer": args.customer or cfg["customer"], "dataset": dataset,
+                "target": config.url(args.target), "protocol": args.protocol or cfg["protocol"],
+                "revision": args.revision, "effort": args.effort,
+                "model": args.model, "environment": args.environment or cfg.get("environment"), "k": args.k}
+        # 响应丢失后的同参数重试复用请求 UUID；成功取得 run_id 后即清除。
+        pending = config.HOME / "pending-create.json"
+        if pending.exists():
+            previous = json.loads(pending.read_text())
+            if previous["server"] != cfg["server"] or previous["spec"] != spec:
+                raise ValueError("存在未确认的创建请求，请先用原参数重试，确认运行编号后再发起新评测")
+            request_id = previous["request_id"]
+        else:
+            request_id = str(uuid4())
+            save(pending, {"server": cfg["server"], "spec": spec, "request_id": request_id})
+        try:
+            result = service.call("POST", "/runs", json=spec | {"request_id": request_id})
+        except ApiError as error:
+            if 400 <= error.status < 500:
+                pending.unlink()
+            raise
+        run_id = result["run_id"]
+        print(f"运行编号：{run_id}；中断后使用 aeval resume {run_id}", flush=True)
+        pending.unlink()
+        return execute(service, cfg, run_id)
+    except (ValueError, httpx.HTTPError, OSError, KeyboardInterrupt) as error:
+        # 不打印 HTTP 异常中的请求 URL、请求头或响应正文，避免泄漏凭据。
+        print(f"评测未完成：{error if isinstance(error, ValueError) else type(error).__name__}")
+        if run_id:
+            print(f"恢复命令：aeval resume {run_id}")
+        return 2
+    finally:
+        if service:
+            service.close()
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

aeval_client/config.py

@@ -0,0 +1,72 @@
+"""交互和手写配置共用 TOML；凭据与恢复文件仅当前用户可读。"""
+
+import getpass
+import json
+import os
+from pathlib import Path
+import tempfile
+import tomllib
+from urllib.parse import urlsplit
+
+HOME = Path.home() / ".aeval"
+
+
+def private_write(path, content):
+    path.parent.mkdir(parents=True, exist_ok=True, mode=0o700)
+    fd, temporary = tempfile.mkstemp(dir=path.parent)
+    try:
+        with os.fdopen(fd, "w") as stream:
+            stream.write(content)
+            stream.flush()
+            os.fsync(stream.fileno())
+        os.replace(temporary, path)
+    finally:
+        if os.path.exists(temporary):
+            os.unlink(temporary)
+
+
+def url(value):
+    value = value if "://" in value else "http://" + value
+    parsed = urlsplit(value)
+    if parsed.scheme not in ("http", "https") or not parsed.hostname or parsed.username or parsed.password or parsed.query or parsed.fragment:
+        raise ValueError("地址必须是 http(s) URL，不能包含凭据、查询参数或片段")
+    return value.rstrip("/")
+
+
+def read(path=None):
+    path = path or HOME / "config"
+    if not path.exists():
+        raise ValueError("请先执行 aeval configure 或创建 ~/.aeval/config")
+    if path.stat().st_mode & 0o077:
+        raise ValueError(f"配置包含凭据，请先执行 chmod 600 {path}")
+    cfg = tomllib.loads(path.read_text())
+    for key in ("server", "api_key", "customer", "protocol"):
+        if not isinstance(cfg.get(key), str) or not cfg[key].strip():
+            raise ValueError(f"配置缺少 {key}")
+    cfg["server"] = url(cfg["server"])
+    if cfg["protocol"] not in ("main", "v1"):
+        raise ValueError("protocol 必须是 main 或 v1")
+    for key in ("dataset", "environment", "agent_token"):
+        if key in cfg and (not isinstance(cfg[key], str) or not cfg[key].strip()):
+            raise ValueError(f"{key} 必须是非空字符串")
+    return cfg
+
+
+def configure():
+    values = {}
+    for key, label in (("server", "Agent-Eval 地址"), ("api_key", "团队 API Key"),
+                       ("customer", "客户"), ("protocol", "Agent 协议 main/v1"),
+                       ("dataset", "默认 Langfuse 数据集 ID（可留空）"),
+                       ("agent_token", "Agent bearer token（可留空）")):
+        values[key] = (getpass.getpass(label + ": ") if "key" in key or "token" in key else input(label + ": ")).strip()
+    values = {k: v for k, v in values.items() if v}
+    content = "".join(f"{k} = {json.dumps(v, ensure_ascii=False)}\n" for k, v in values.items())
+    # 先验证临时配置，避免无效交互覆盖已有配置。
+    temporary = HOME / "config.new"
+    private_write(temporary, content)
+    try:
+        read(temporary)
+        temporary.replace(HOME / "config")
+    finally:
+        temporary.unlink(missing_ok=True)
+    print(f"配置已保存：{HOME / 'config'}")

aidong_aeval-0.1.0.dist-info/METADATA

@@ -0,0 +1,56 @@
+Metadata-Version: 2.5
+Name: aidong-aeval
+Version: 0.1.0
+Summary: OmniAgent 开发者评测客户端
+Requires-Python: >=3.11
+Requires-Dist: httpx<1,>=0.27
+Description-Content-Type: text/markdown
+
+# Agent-Eval 开发者 CLI
+
+在开发机调用 OmniAgent，中心服务统一出题、判定和保存报告。
+Agent 自行配置 Langfuse 和写工具禁用；客户端不需要数据库或 Langfuse 凭据。
+
+```bash
+uv tool install aidong-aeval
+aeval configure
+aeval datasets
+aeval --target 127.0.0.1:8787 --dataset <数据集ID> --revision <版本>
+aeval resume <运行编号>
+aeval results <运行编号>
+```
+
+PyPI 包名为 `aidong-aeval`，命令名为 `aeval`。
+开发安装：`uv tool install ./client`。旧运维包也提供 aeval 命令，请使用不同工具环境。
+
+也可手写 `~/.aeval/config`（TOML），并执行 `chmod 600 ~/.aeval/config`：
+
+```toml
+server = "https://评测服务域名"
+api_key = "团队口令"
+customer = "客户标识"
+protocol = "v1"
+```
+
+可选 `dataset`、`environment`、`agent_token`。环境无默认值；Agent bearer token
+仅发给 Agent，不发给评测服务。`--environment` 只核对、不设置环境；
+`--model` 只核对实际模型，不切换模型。v1 可传 `--effort high`，
+CLI 先验证模型支持的档位；main 不支持该参数。
+
+调用断线后先执行 resume，从 trace 恢复已有回答；没有完成证据时不自动重问。
+确认允许重复调用时执行 `aeval resume <编号> --retry-uncertain`，重新预检后使用新会话。
+回答缓存保存在 `~/.aeval/runs/`，确认上传后删除。
+
+## 发布准备
+
+```bash
+uv build --project client --out-dir dist/client
+uvx twine check dist/client/*
+```
+
+检查 wheel/sdist 只包含客户端源码和公开说明；在隔离环境安装 wheel 并运行
+`aeval --help`，完成连接真实测试服务的验证。凭据、客户数据与服务端源码不打包。
+人工确认 PyPI 项目名称、维护者权限与发布凭据后才可执行上传；
+本仓不配置 tag 自动发布。
+首次上传创建 PyPI 项目；后续发布使用仅限 `aidong-aeval` 项目的 Token，
+不长期保留首次创建项目使用的账户级 Token。

aidong_aeval-0.1.0.dist-info/RECORD

@@ -0,0 +1,8 @@
+aeval_client/__init__.py,sha256=imxMk1yQiH3iHPyY4goErSC1H0HoHDBjQNLw7ZIVe-E,71
+aeval_client/agent.py,sha256=C05cEnzlj67-ndHkeSqlXD8Ozmrh6X5bE47AJEsmnkU,2414
+aeval_client/cli.py,sha256=MPLDpB3QmB81aO0kSYJx73uY96lc0k-YUx9h1atTJ5U,9944
+aeval_client/config.py,sha256=IWQmg_1V1Qtj4Ct_G7Vb7g6Jk_OhW4BwNJkonX-NZZY,2885
+aidong_aeval-0.1.0.dist-info/METADATA,sha256=ZI8R2S7GzsGbiSKXoc__TjVuG9WjMsMupij-djyMcJU,2187
+aidong_aeval-0.1.0.dist-info/WHEEL,sha256=W3fkpkm7-wf9vBI5Z-7s0eWkeM-spu78I8Neb98DeEg,87
+aidong_aeval-0.1.0.dist-info/entry_points.txt,sha256=4mrxz4u-qMm7FdgIovKnB5yR4c9lEKGLYgV5bhNbKE0,48
+aidong_aeval-0.1.0.dist-info/RECORD,,

aidong_aeval-0.1.0.dist-info/WHEEL

@@ -0,0 +1,4 @@
+Wheel-Version: 1.0
+Generator: hatchling 1.32.4
+Root-Is-Purelib: true
+Tag: py3-none-any

aidong_aeval-0.1.0.dist-info/entry_points.txt

@@ -0,0 +1,2 @@
+[console_scripts]
+aeval = aeval_client.cli:main