agentengine-sdk-python 0.7.0__py3-none-any.whl → 0.8.1__py3-none-any.whl

ksadk/cli/cmd_eval.py

@@ -0,0 +1,386 @@
+"""agentengine eval - stable CLI shell for local evaluation."""
+
+from __future__ import annotations
+
+import asyncio
+from pathlib import Path
+
+import click
+from rich.markup import escape
+from rich.padding import Padding
+from rich.table import Table
+
+from ksadk.cli.ui import (
+    configure_ui_runtime,
+    emit_json,
+    get_console,
+    is_json_output,
+    new_table,
+    print_info,
+    print_kv,
+    print_success,
+    print_title,
+    status_rich_style,
+)
+from ksadk.evaluation import (
+    EvaluationConfig,
+    EvaluationExecutionError,
+    EvaluationNotImplementedError,
+    EvaluationRequest,
+    TargetRef,
+    execute_evaluation,
+    load_evalset,
+)
+from ksadk.evaluation.contracts import (
+    DataPolicy,
+    EvalRunReport,
+    EvalRunStatus,
+    EvalRunSummary,
+    MetricResult,
+    MetricStatus,
+    TargetKind,
+    TargetRunStatus,
+)
+from ksadk.evaluation.evalset import EvalSetParseError
+from ksadk.evaluation.evaluators import DEFAULT_EVALUATORS, SUPPORTED_EVALUATORS
+from ksadk.evaluation.storage import EvaluationStorage
+
+_DEFAULT_EVALUATORS = tuple(DEFAULT_EVALUATORS)
+_DATA_POLICIES = tuple(policy.value for policy in DataPolicy)
+
+
+class EvaluationCliError(click.ClickException):
+    exit_code = 2
+
+
+@click.command(context_settings=dict(help_option_names=["-h", "--help"]))
+@click.option(
+    "--evalset-file",
+    required=True,
+    type=click.Path(exists=True, dir_okay=False, path_type=Path),
+    help="本地 EvalSet YAML/JSON 文件",
+)
+@click.option(
+    "--agent-dir",
+    type=click.Path(exists=True, file_okay=False, path_type=Path),
+    help="本地 Agent 项目目录",
+)
+@click.option("--a2a-url", type=str, help="远端 A2A Agent Card URL")
+@click.option(
+    "--codex-worktree",
+    type=click.Path(exists=True, file_okay=False, path_type=Path),
+    help="Codex 评测使用的 Git worktree",
+)
+@click.option("--entrypoint", type=str, help="本地 Agent 入口；缺省时由 target adapter 检测")
+@click.option("--credential-ref", type=str, help="A2A 鉴权引用；不会写入凭据值")
+@click.option("--codex-profile", type=str, help="Codex 受信评测 profile 名称")
+@click.option(
+    "--evaluator",
+    "evaluators",
+    multiple=True,
+    type=click.Choice(SUPPORTED_EVALUATORS),
+    help="启用评估器；可重复指定",
+)
+@click.option("--judge-model", type=str, help="LLM Judge 模型名")
+@click.option("--judge-api-base", type=str, help="LLM Judge OpenAI 兼容 API 地址")
+@click.option(
+    "--judge-api-key-env",
+    default="KSADK_EVAL_JUDGE_API_KEY",
+    show_default=True,
+    type=str,
+    help="保存 Judge API Key 的环境变量名",
+)
+@click.option(
+    "--timeout-seconds",
+    default=120,
+    show_default=True,
+    type=click.IntRange(1, 3600),
+    help="单个 Case 超时时间（秒）",
+)
+@click.option("--fail-fast", is_flag=True, help="首个失败 Case 后停止")
+@click.option(
+    "--data-policy",
+    type=click.Choice(_DATA_POLICIES),
+    default="local_only",
+    show_default=True,
+    help="评测数据策略",
+)
+@click.option(
+    "--report-dir",
+    type=click.Path(file_okay=False, path_type=Path),
+    help="报告目录；缺省为项目 .agentkit/evaluations",
+)
+@click.option("--validate-only", is_flag=True, help="只识别并校验 EvalSet 和 CLI 参数")
+@click.option(
+    "--format",
+    "output_format",
+    type=click.Choice(["pretty", "json"]),
+    help="终端输出格式",
+)
+def eval(
+    evalset_file: Path,
+    agent_dir: Path | None,
+    a2a_url: str | None,
+    codex_worktree: Path | None,
+    entrypoint: str | None,
+    credential_ref: str | None,
+    codex_profile: str | None,
+    evaluators: tuple[str, ...],
+    judge_model: str | None,
+    judge_api_base: str | None,
+    judge_api_key_env: str,
+    timeout_seconds: int,
+    fail_fast: bool,
+    data_policy: str,
+    report_dir: Path | None,
+    validate_only: bool,
+    output_format: str | None,
+) -> None:
+    """校验 EvalSet，并通过统一入口发起 Agent 评测。"""
+
+    if output_format:
+        configure_ui_runtime(output_mode=output_format)
+    target = _target_ref(
+        agent_dir=agent_dir,
+        a2a_url=a2a_url,
+        codex_worktree=codex_worktree,
+        entrypoint=entrypoint,
+        credential_ref=credential_ref,
+        codex_profile=codex_profile,
+    )
+    request = _build_request(
+        evalset_file=evalset_file,
+        target=target,
+        evaluators=evaluators,
+        judge_model=judge_model,
+        judge_api_base=judge_api_base,
+        judge_api_key_env=judge_api_key_env,
+        timeout_seconds=timeout_seconds,
+        fail_fast=fail_fast,
+        data_policy=data_policy,
+        report_dir=report_dir,
+    )
+    if validate_only:
+        _render_validation(request)
+        return
+
+    report = _execute_request(request)
+    exit_code = _report_exit_code(report)
+    _render_report(report, report_dir=request.report_dir)
+    if exit_code:
+        raise click.exceptions.Exit(exit_code)
+
+
+def _build_request(
+    *,
+    evalset_file: Path,
+    target: TargetRef,
+    evaluators: tuple[str, ...],
+    judge_model: str | None,
+    judge_api_base: str | None,
+    judge_api_key_env: str,
+    timeout_seconds: int,
+    fail_fast: bool,
+    data_policy: str,
+    report_dir: Path | None,
+) -> EvaluationRequest:
+    try:
+        evalset = load_evalset(evalset_file)
+    except EvalSetParseError as exc:
+        raise click.UsageError(f"{exc.code}: {exc}") from exc
+
+    return EvaluationRequest(
+        evalset=evalset,
+        target=target,
+        config=EvaluationConfig(
+            timeout_seconds=timeout_seconds,
+            fail_fast=fail_fast,
+            evaluators=list(evaluators) or list(_DEFAULT_EVALUATORS),
+            data_policy=data_policy,
+            judge_model=judge_model,
+            judge_api_base=judge_api_base,
+            judge_api_key_env=judge_api_key_env,
+        ),
+        report_dir=str((report_dir or Path.cwd() / ".agentkit/evaluations").resolve()),
+    )
+
+
+def _execute_request(request: EvaluationRequest) -> EvalRunReport:
+    on_case_started = None
+    if not is_json_output():
+        _render_start(request)
+        on_case_started = _render_case_started
+    try:
+        return asyncio.run(execute_evaluation(request, on_case_started=on_case_started))
+    except (EvaluationNotImplementedError, EvaluationExecutionError) as exc:
+        raise EvaluationCliError(str(exc)) from exc
+
+
+def _render_report(report: EvalRunReport, *, report_dir: str | None = None) -> None:
+    if is_json_output():
+        emit_json(report.model_dump(mode="json", by_alias=True, exclude_none=True))
+    else:
+        print_title("Agent 评测完成")
+        get_console().print("[title][i]评测摘要[/i][/]")
+        print_kv("运行状态", report.status.value)
+        print_kv("Run ID", report.spec.id)
+        if report_dir:
+            print_kv(
+                "报告文件",
+                str(EvaluationStorage(report_dir).report_path(report.spec.id)),
+                value_style="#58a6ff",
+            )
+        _render_report_preview(report)
+
+
+
+def _render_start(request: EvaluationRequest) -> None:
+    print_info(
+        "开始评测："
+        f"{request.evalset.name}，{len(request.evalset.cases)} 个 Case，"
+        f"Target: {request.target.kind.value}"
+    )
+
+
+def _render_case_started(case_id: str, index: int, total_cases: int) -> None:
+    print_info(f"[{index}/{total_cases}] 执行 Case: {case_id}")
+
+
+def _render_report_preview(report: EvalRunReport) -> None:
+    _render_summary(report.summary)
+    _render_case_table(report)
+    remaining_cases = len(report.case_runs) - 5
+    if remaining_cases > 0:
+        print_info(f"其余 {remaining_cases} 个 Case 已省略")
+
+
+def _render_summary(summary: EvalRunSummary) -> None:
+    items = (
+        ("总计", summary.total_cases, "white"),
+        ("通过", summary.passed_cases, "ok"),
+        ("失败", summary.failed_cases, "err"),
+        ("错误", summary.error_cases, "err"),
+        ("不可用", summary.unavailable_cases, "warn"),
+        ("取消", summary.cancelled_cases, "warn"),
+    )
+    table = Table.grid(padding=(0, 2))
+    for _label, _count, style in items:
+        table.add_column(justify="center", style=style, no_wrap=True)
+    table.add_row(*(label for label, _count, _style in items), style="muted")
+    table.add_row(*(f"[{style if count else 'muted'}]{count}[/]" for _label, count, style in items))
+    print_info("  结果统计")
+    get_console().print(Padding(table, (0, 0, 0, 4)))
+
+
+def _render_case_table(report: EvalRunReport) -> None:
+    table = new_table("  评测集列表")
+    table.add_column("Case", style="#58a6ff", no_wrap=True)
+    table.add_column("目标状态", no_wrap=True)
+    table.add_column("耗时", justify="right", no_wrap=True)
+    table.add_column("指标")
+    for case_run in report.case_runs[:5]:
+        target_status = case_run.target_run.status.value
+        table.add_row(
+            escape(case_run.case_id),
+            f"[{status_rich_style(target_status)}]{target_status}[/]",
+            _format_duration(case_run.target_run.duration_ms),
+            _metric_summary(case_run.metrics),
+        )
+    get_console().print(table)
+
+
+def _format_duration(duration_ms: int | None) -> str:
+    return f"{duration_ms} ms" if duration_ms is not None else "-"
+
+
+def _metric_summary(metrics: list[MetricResult]) -> str:
+    if not metrics:
+        return "无指标"
+    counts = {status: sum(metric.status is status for metric in metrics) for status in MetricStatus}
+    return "，".join(f"{status.value} {count}" for status, count in counts.items() if count)
+
+
+def _report_exit_code(report: EvalRunReport) -> int:
+    if report.status in {
+        EvalRunStatus.ERROR,
+        EvalRunStatus.CANCELLED,
+        EvalRunStatus.PENDING,
+        EvalRunStatus.RUNNING,
+    }:
+        return 2
+    if any(
+        case_run.target_run.status is TargetRunStatus.UNAVAILABLE
+        or any(
+            metric.required and metric.status is MetricStatus.UNAVAILABLE
+            for metric in case_run.metrics
+        )
+        for case_run in report.case_runs
+    ):
+        return 3
+    return 1 if report.status is EvalRunStatus.FAILED else 0
+
+
+def _target_ref(
+    agent_dir: Path | None,
+    a2a_url: str | None,
+    codex_worktree: Path | None,
+    entrypoint: str | None,
+    credential_ref: str | None,
+    codex_profile: str | None,
+) -> TargetRef:
+    targets = [agent_dir is not None, bool(a2a_url), codex_worktree is not None]
+    if sum(targets) != 1:
+        raise click.UsageError("--agent-dir、--a2a-url、--codex-worktree 必须且只能指定一个")
+    if credential_ref and not a2a_url:
+        raise click.UsageError("--credential-ref 只能与 --a2a-url 一起使用")
+    if entrypoint and not agent_dir:
+        raise click.UsageError("--entrypoint 只能与 --agent-dir 一起使用")
+    if codex_profile and not codex_worktree:
+        raise click.UsageError("--codex-profile 只能与 --codex-worktree 一起使用")
+    if agent_dir:
+        return TargetRef(
+            kind=TargetKind.LOCAL_SOURCE,
+            locator=str(agent_dir.resolve()),
+            entrypoint=entrypoint,
+        )
+    if a2a_url:
+        return TargetRef(
+            kind=TargetKind.A2A,
+            locator=a2a_url,
+            credential_ref=credential_ref,
+        )
+    assert codex_worktree is not None
+    return TargetRef(
+        kind=TargetKind.CODEX_WORKTREE,
+        locator=str(codex_worktree.resolve()),
+        profile=codex_profile,
+    )
+
+
+def _render_validation(request: EvaluationRequest) -> None:
+    payload = {
+        "valid": True,
+        "evalset": {
+            "name": request.evalset.name,
+            "sourceFormat": request.evalset.source_format,
+            "caseCount": len(request.evalset.cases),
+            "contentDigest": request.evalset.content_digest,
+        },
+        "target": request.target.model_dump(mode="json", by_alias=True, exclude_none=True),
+        "config": request.config.model_dump(mode="json", by_alias=True),
+        "reportDir": request.report_dir,
+    }
+    if is_json_output():
+        emit_json(payload)
+        return
+    print_title("评测配置校验")
+    print_kv("EvalSet", request.evalset.name)
+    print_kv("格式", request.evalset.source_format)
+    print_kv("Case 数量", str(len(request.evalset.cases)))
+    print_kv("内容摘要", request.evalset.content_digest)
+    print_kv("Target", f"{request.target.kind.value}: {request.target.locator}")
+    print_success("EvalSet 与 CLI 参数有效")
+
+
+# Eval reports use --report-dir exclusively; do not inject the generic child --output.
+eval.disable_global_output_option = True

ksadk/cli/cmd_files.py

@@ -54,7 +54,7 @@ def _format_size(size_bytes: object | None) -> str:
     if size_bytes is None:
         return "-"
     try:
-        value = int(size_bytes)
+        value = int(str(size_bytes))
     except (TypeError, ValueError):
         return str(size_bytes)
     if value < 1024:
@@ -110,7 +110,9 @@ def _echo_section_title(title: str, *, fg: str = "bright_blue") -> None:
     click.secho(title, fg=fg, bold=True)
 
 
-def _normalize_entry_payload(entry: dict, *, root_label: str, workspace_real_root: str | None = None) -> dict:
+def _normalize_entry_payload(
+    entry: dict, *, root_label: str, workspace_real_root: str | None = None
+) -> dict:
     normalized = dict(entry or {})
     path = str(normalized.get("path") or "").strip()
     normalized["display_path"] = _workspace_display_path(path, root_label=root_label)
@@ -125,7 +127,10 @@ def _normalize_entry_payload(entry: dict, *, root_label: str, workspace_real_roo
 
 def _build_list_payload(payload: dict) -> dict:
     root_label = _workspace_root_label(payload)
-    workspace_real_root = str(payload.get("workspace_real_root") or payload.get("workspace_path") or "").strip() or None
+    workspace_real_root = (
+        str(payload.get("workspace_real_root") or payload.get("workspace_path") or "").strip()
+        or None
+    )
     entries = [
         _normalize_entry_payload(
             entry,
@@ -142,7 +147,9 @@ def _build_list_payload(payload: dict) -> dict:
             "ok": True,
             "action": "list",
             "workspace_root": root_label,
-            "workspace_display_path": _workspace_display_path(payload.get("path", "."), root_label=root_label),
+            "workspace_display_path": _workspace_display_path(
+                payload.get("path", "."), root_label=root_label
+            ),
             "workspace_real_root": workspace_real_root,
             "workspace_real_path": _workspace_real_path(
                 payload.get("path", "."),
@@ -182,7 +189,9 @@ def _build_upload_payload(payload: dict, *, local_path: Path, remote_path: str)
             "local_path": str(local_path),
             "requested_remote_path": remote_path,
             "remote_path": resolved_remote_path,
-            "remote_display_path": _workspace_display_path(resolved_remote_path, root_label=root_label),
+            "remote_display_path": _workspace_display_path(
+                resolved_remote_path, root_label=root_label
+            ),
             "entry": normalized_entry,
             "summary": {
                 "uploaded": 1,
@@ -268,7 +277,9 @@ def _build_sync_payload(payload: dict) -> dict:
             "ok": True,
             "action": str(payload.get("direction", "sync")).strip().lower() or "sync",
             "workspace_root": root_label,
-            "remote_display_path": _workspace_display_path(payload.get("remote_path", "."), root_label=root_label),
+            "remote_display_path": _workspace_display_path(
+                payload.get("remote_path", "."), root_label=root_label
+            ),
             "transport_mode": transport_mode or None,
             "transport_hint": transport_hint,
             "summary": {
@@ -304,20 +315,27 @@ def _emit_payload(payload, output_mode: str | None) -> None:
         if not entries:
             click.secho("当前目录为空。", fg="bright_black")
             return
-        directories = payload.get("directories") or [entry for entry in entries if entry.get("type") == "directory"]
-        files = payload.get("files") or [entry for entry in entries if entry.get("type") != "directory"]
+        directories = payload.get("directories") or [
+            entry for entry in entries if entry.get("type") == "directory"
+        ]
+        files = payload.get("files") or [
+            entry for entry in entries if entry.get("type") != "directory"
+        ]
         if directories:
             _echo_section_title(f"目录（{len(directories)}）", fg="cyan")
             for entry in directories:
-                click.secho(f"  {entry.get('display_path') or _workspace_display_path(entry.get('path', ''), root_label=root_label)}", fg="cyan")
+                display_path = entry.get("display_path") or _workspace_display_path(
+                    entry.get("path", ""), root_label=root_label
+                )
+                click.secho(f"  {display_path}", fg="cyan")
         if files:
             _echo_section_title(f"文件（{len(files)}）", fg="green")
             for entry in files:
-                click.secho(
-                    f"  {entry.get('display_path') or _workspace_display_path(entry.get('path', ''), root_label=root_label)}"
-                    f"  {entry.get('size_human') or _format_size(entry.get('size_bytes'))}",
-                    fg="green",
+                display_path = entry.get("display_path") or _workspace_display_path(
+                    entry.get("path", ""), root_label=root_label
                 )
+                size_text = entry.get("size_human") or _format_size(entry.get("size_bytes"))
+                click.secho(f"  {display_path}  {size_text}", fg="green")
         return
     click.echo(str(payload))
 
@@ -469,11 +487,13 @@ def _collect_local_files(
     ignore_dev_artifacts: bool = False,
     ignore_git_artifacts: bool = True,
 ) -> list[tuple[str, Path]]:
-    return _collect_local_files_report(
+    report = _collect_local_files_report(
         local_dir,
         ignore_dev_artifacts=ignore_dev_artifacts,
         ignore_git_artifacts=ignore_git_artifacts,
-    )["files"]
+    )
+    files = report.get("files")
+    return list(files) if isinstance(files, list) else []
 
 
 def _workspace_client_kwargs(
@@ -505,7 +525,12 @@ def _state_prefers_action_proxy(state: dict) -> bool:
 
 
 def _resolve_workspace_region(region: str | None, state: dict) -> str:
-    return _normalize(region) or _normalize(state.get("region")) or os.getenv("KSYUN_REGION") or DEFAULT_REGION
+    return (
+        _normalize(region)
+        or _normalize(state.get("region"))
+        or os.getenv("KSYUN_REGION")
+        or DEFAULT_REGION
+    )
 
 
 def _resolve_workspace_agent_ref(explicit_ref: str | None, cwd: Path) -> str | None:
@@ -674,7 +699,7 @@ async def _download_workspace_file(
             kwargs["endpoint"] = endpoint
         if api_key:
             kwargs["api_key"] = api_key
-        return await client.download_workspace_file(**kwargs)
+        return bytes(await client.download_workspace_file(**kwargs))
 
 
 async def _delete_workspace_file(
@@ -917,7 +942,9 @@ def files():
 @click.option("--api-key", help="Runtime API Key (与 --endpoint 搭配使用)")
 @click.option("--path", default=".", show_default=True, help="Workspace 目录路径")
 @click.option("--recursive", is_flag=True, help="递归列出目录")
-@click.option("--region", "-r", default=None, envvar=None, help="区域 (默认优先读取 .agentengine.state)")
+@click.option(
+    "--region", "-r", default=None, envvar=None, help="区域 (默认优先读取 .agentengine.state)"
+)
 @cli_output_option()
 def list_files(
     agent_ref: str | None,
@@ -965,7 +992,9 @@ def list_files(
     help="本地文件路径",
 )
 @click.option("--remote-path", required=True, help="Workspace 目标路径")
-@click.option("--region", "-r", default=None, envvar=None, help="区域 (默认优先读取 .agentengine.state)")
+@click.option(
+    "--region", "-r", default=None, envvar=None, help="区域 (默认优先读取 .agentengine.state)"
+)
 @cli_output_option()
 def upload_file(
     agent_ref: str | None,
@@ -1029,7 +1058,9 @@ def upload_file(
     required=True,
     help="本地输出文件路径",
 )
-@click.option("--region", "-r", default=None, envvar=None, help="区域 (默认优先读取 .agentengine.state)")
+@click.option(
+    "--region", "-r", default=None, envvar=None, help="区域 (默认优先读取 .agentengine.state)"
+)
 @cli_output_option()
 def download_file(
     agent_ref: str | None,
@@ -1083,7 +1114,9 @@ def download_file(
 @click.option("--api-key", help="Runtime API Key (与 --endpoint 搭配使用)")
 @click.option("--remote-path", required=True, help="Workspace 文件路径")
 @click.option("--yes", is_flag=True, help="跳过确认")
-@click.option("--region", "-r", default=None, envvar=None, help="区域 (默认优先读取 .agentengine.state)")
+@click.option(
+    "--region", "-r", default=None, envvar=None, help="区域 (默认优先读取 .agentengine.state)"
+)
 @cli_output_option()
 def delete_file(
     agent_ref: str | None,
@@ -1105,7 +1138,9 @@ def delete_file(
         api_key=api_key,
         region=region,
     )
-    if not yes and not click.confirm(f"删除 workspace 文件 {normalized_remote_path}?", default=False):
+    if not yes and not click.confirm(
+        f"删除 workspace 文件 {normalized_remote_path}?", default=False
+    ):
         raise click.Abort()
     payload = asyncio.run(
         _delete_workspace_file(
@@ -1137,7 +1172,9 @@ def delete_file(
 )
 @click.option("--remote-path", default=".", show_default=True, help="Workspace 目标目录")
 @click.option("--force", is_flag=True, help="覆盖远端已存在的同名文件")
-@click.option("--region", "-r", default=None, envvar=None, help="区域 (默认优先读取 .agentengine.state)")
+@click.option(
+    "--region", "-r", default=None, envvar=None, help="区域 (默认优先读取 .agentengine.state)"
+)
 @cli_output_option()
 def push_files(
     agent_ref: str | None,
@@ -1188,7 +1225,9 @@ def push_files(
     help="本地输出目录",
 )
 @click.option("--force", is_flag=True, help="覆盖本地已存在的同名文件")
-@click.option("--region", "-r", default=None, envvar=None, help="区域 (默认优先读取 .agentengine.state)")
+@click.option(
+    "--region", "-r", default=None, envvar=None, help="区域 (默认优先读取 .agentengine.state)"
+)
 @cli_output_option()
 def pull_files(
     agent_ref: str | None,