agentengine-sdk-python 0.8.1__py3-none-any.whl → 0.8.2__py3-none-any.whl

ksadk/cli/cmd_eval.py

@@ -31,6 +31,11 @@ from ksadk.evaluation import (
     execute_evaluation,
     load_evalset,
 )
+from ksadk.evaluation.agent_eval_client import (
+    AgentEvalCloudClientError,
+    AgentEvalCloudDatasetClient,
+)
+from ksadk.evaluation.cloud_service import CloudEvalSetPreviewError, CloudEvalSetService
 from ksadk.evaluation.contracts import (
     DataPolicy,
     EvalRunReport,
@@ -42,10 +47,9 @@ from ksadk.evaluation.contracts import (
     TargetRunStatus,
 )
 from ksadk.evaluation.evalset import EvalSetParseError
-from ksadk.evaluation.evaluators import DEFAULT_EVALUATORS, SUPPORTED_EVALUATORS
+from ksadk.evaluation.evaluators import SUPPORTED_EVALUATORS, resolve_evaluator_plan
 from ksadk.evaluation.storage import EvaluationStorage
 
-_DEFAULT_EVALUATORS = tuple(DEFAULT_EVALUATORS)
 _DATA_POLICIES = tuple(policy.value for policy in DataPolicy)
 
 
@@ -56,10 +60,13 @@ class EvaluationCliError(click.ClickException):
 @click.command(context_settings=dict(help_option_names=["-h", "--help"]))
 @click.option(
     "--evalset-file",
-    required=True,
+    required=False,
     type=click.Path(exists=True, dir_okay=False, path_type=Path),
     help="本地 EvalSet YAML/JSON 文件",
 )
+@click.option("--dataset-id", type=str, help="云端 Dataset ID；必须配合固定版本使用")
+@click.option("--dataset-version", type=click.IntRange(1), help="云端 Dataset immutable version")
+@click.option("--dataset-project-id", type=str, help="云端 Dataset 所属项目 ID")
 @click.option(
     "--agent-dir",
     type=click.Path(exists=True, file_okay=False, path_type=Path),
@@ -119,6 +126,9 @@ class EvaluationCliError(click.ClickException):
 )
 def eval(
     evalset_file: Path,
+    dataset_id: str | None,
+    dataset_version: int | None,
+    dataset_project_id: str | None,
     agent_dir: Path | None,
     a2a_url: str | None,
     codex_worktree: Path | None,
@@ -150,6 +160,9 @@ def eval(
     )
     request = _build_request(
         evalset_file=evalset_file,
+        dataset_id=dataset_id,
+        dataset_version=dataset_version,
+        dataset_project_id=dataset_project_id,
         target=target,
         evaluators=evaluators,
         judge_model=judge_model,
@@ -173,7 +186,10 @@ def eval(
 
 def _build_request(
     *,
-    evalset_file: Path,
+    evalset_file: Path | None,
+    dataset_id: str | None,
+    dataset_version: int | None,
+    dataset_project_id: str | None,
     target: TargetRef,
     evaluators: tuple[str, ...],
     judge_model: str | None,
@@ -184,10 +200,37 @@ def _build_request(
     data_policy: str,
     report_dir: Path | None,
 ) -> EvaluationRequest:
-    try:
-        evalset = load_evalset(evalset_file)
-    except EvalSetParseError as exc:
-        raise click.UsageError(f"{exc.code}: {exc}") from exc
+    cloud_dataset = None
+    if dataset_id:
+        if evalset_file is not None:
+            raise click.UsageError("--evalset-file 与 --dataset-id 不能同时使用")
+        if dataset_version is None:
+            raise click.UsageError("--dataset-id 必须同时指定 --dataset-version")
+        try:
+            service = CloudEvalSetService(
+                Path.cwd(),
+                AgentEvalCloudDatasetClient(),
+            )
+            pulled = asyncio.run(
+                service.pull(
+                    dataset_id=dataset_id,
+                    version=dataset_version,
+                    project_id=dataset_project_id,
+                )
+            )
+        except (AgentEvalCloudClientError, CloudEvalSetPreviewError, ValueError) as exc:
+            raise click.UsageError(str(exc)) from exc
+        evalset = pulled.evalset
+        cloud_dataset = pulled.cloud_dataset
+    else:
+        if evalset_file is None:
+            raise click.UsageError("必须指定 --evalset-file 或 --dataset-id")
+        if dataset_version is not None or dataset_project_id:
+            raise click.UsageError("云端 Dataset 参数必须与 --dataset-id 一起使用")
+        try:
+            evalset = load_evalset(evalset_file)
+        except EvalSetParseError as exc:
+            raise click.UsageError(f"{exc.code}: {exc}") from exc
 
     return EvaluationRequest(
         evalset=evalset,
@@ -195,13 +238,14 @@ def _build_request(
         config=EvaluationConfig(
             timeout_seconds=timeout_seconds,
             fail_fast=fail_fast,
-            evaluators=list(evaluators) or list(_DEFAULT_EVALUATORS),
+            evaluators=list(evaluators),
             data_policy=data_policy,
             judge_model=judge_model,
             judge_api_base=judge_api_base,
             judge_api_key_env=judge_api_key_env,
         ),
         report_dir=str((report_dir or Path.cwd() / ".agentkit/evaluations").resolve()),
+        cloud_dataset=cloud_dataset,
     )
 
 
@@ -358,6 +402,14 @@ def _target_ref(
 
 
 def _render_validation(request: EvaluationRequest) -> None:
+    try:
+        evaluation_plan = resolve_evaluator_plan(
+            request.evalset.cases,
+            request.config.evaluators,
+            request.config,
+        )
+    except ValueError as exc:
+        raise click.UsageError(str(exc)) from exc
     payload = {
         "valid": True,
         "evalset": {
@@ -368,8 +420,13 @@ def _render_validation(request: EvaluationRequest) -> None:
         },
         "target": request.target.model_dump(mode="json", by_alias=True, exclude_none=True),
         "config": request.config.model_dump(mode="json", by_alias=True),
+        "evaluationPlan": evaluation_plan,
         "reportDir": request.report_dir,
     }
+    if request.cloud_dataset is not None:
+        payload["cloudDataset"] = request.cloud_dataset.model_dump(
+            mode="json", by_alias=True, exclude_none=True
+        )
     if is_json_output():
         emit_json(payload)
         return

ksadk/cli/cmd_evalset.py

@@ -0,0 +1,284 @@
+"""Commands for inspecting and publishing immutable cloud EvalSet snapshots."""
+
+from __future__ import annotations
+
+import asyncio
+import json
+from pathlib import Path
+
+import click
+import yaml
+
+from ksadk.evaluation.agent_eval_client import (
+    AgentEvalCloudClientError,
+    AgentEvalCloudDatasetClient,
+)
+from ksadk.evaluation.cloud_service import CloudEvalSetPreviewError, CloudEvalSetService
+from ksadk.evaluation.contracts import DataPolicy
+from ksadk.evaluation.evalset import EvalSetParseError, load_evalset, parse_evalset
+
+_DATA_POLICIES = tuple(policy.value for policy in DataPolicy)
+_TEMPLATE_NAMES = ("knowledge-qa", "structured-output", "tool-routing", "service-sla")
+_TEMPLATES: dict[str, dict] = {
+    "knowledge-qa": {
+        "schemaVersion": "ksadk.eval/v1",
+        "name": "knowledge-qa",
+        "cases": [
+            {
+                "id": "capital",
+                "input": "中国的首都是哪里？",
+                "reference_output": "北京",
+            }
+        ],
+    },
+    "structured-output": {
+        "schemaVersion": "ksadk.eval/v1",
+        "name": "structured-output",
+        "cases": [
+            {
+                "id": "extract-order",
+                "input": "从‘订单 A123，金额 99 元’提取订单信息，并只返回 JSON。",
+                "assertions": [
+                    {
+                        "type": "response.jsonSchema",
+                        "value": {
+                            "type": "object",
+                            "required": ["orderId", "amount"],
+                            "properties": {
+                                "orderId": {"type": "string"},
+                                "amount": {"type": "number"},
+                            },
+                        },
+                    }
+                ],
+            }
+        ],
+    },
+    "tool-routing": {
+        "schemaVersion": "ksadk.eval/v1",
+        "name": "tool-routing",
+        "cases": [
+            {
+                "id": "weather-lookup",
+                "input": "查询北京明天的天气，并给出出行建议。",
+                "reference_output": "根据天气查询结果回答北京明天的天气，并给出出行建议。",
+                "expectedTools": [{"name": "weather_lookup"}],
+                "assertions": [
+                    {"type": "tool.succeeded", "value": "weather_lookup"},
+                    {"type": "tool.sequence", "value": ["weather_lookup"]},
+                ],
+            }
+        ],
+    },
+    "service-sla": {
+        "schemaVersion": "ksadk.eval/v1",
+        "name": "service-sla",
+        "cases": [
+            {
+                "id": "password-reset",
+                "input": "如何重置密码？",
+                "reference_output": "可通过登录页的忘记密码入口重置密码。",
+                "assertions": [
+                    {"type": "runtime.maxLatencyMs", "value": 3000},
+                    {"type": "runtime.maxTotalTokens", "value": 300},
+                ],
+            }
+        ],
+    },
+}
+
+
+def _render(value: dict, output_format: str) -> None:
+    if output_format == "json":
+        click.echo(json.dumps(value, ensure_ascii=False, sort_keys=True))
+        return
+    for key, item in value.items():
+        click.echo(f"{key}: {item}")
+
+
+def _load_snapshot(evalset_file: Path, data_policy: str):
+    try:
+        evalset = load_evalset(evalset_file)
+    except EvalSetParseError as exc:
+        raise click.UsageError(f"{exc.code}: {exc}") from exc
+    service = CloudEvalSetService(Path.cwd(), client=_PreviewOnlyCloudClient())
+    try:
+        return evalset, service.preview(evalset, data_policy=DataPolicy(data_policy))
+    except CloudEvalSetPreviewError as exc:
+        raise click.UsageError(str(exc)) from exc
+
+
+class _PreviewOnlyCloudClient:
+    async def publish_snapshot(self, *args, **kwargs):  # pragma: no cover - preview never publishes
+        raise RuntimeError("preview does not publish")
+
+
+@click.group()
+def evalset() -> None:
+    """Inspect or publish versioned cloud EvalSet snapshots."""
+
+
+@evalset.command("init")
+@click.option("--template", "template_name", type=click.Choice(_TEMPLATE_NAMES), required=True)
+@click.option("--output-file", required=True, type=click.Path(dir_okay=False, path_type=Path))
+@click.option("--force", is_flag=True, help="覆盖已有文件")
+@click.option("--format", "output_format", type=click.Choice(["pretty", "json"]), default="pretty")
+def init(template_name: str, output_file: Path, force: bool, output_format: str) -> None:
+    """Create a native EvalSet template without accessing cloud services."""
+
+    template = _TEMPLATES[template_name]
+    try:
+        parse_evalset(template)
+    except EvalSetParseError as exc:  # pragma: no cover - protects static templates
+        raise click.ClickException(f"内置模板无效: {exc}") from exc
+
+    output = output_file.expanduser().resolve()
+    if output.exists() and not force:
+        raise click.UsageError("输出文件已存在；如需覆盖请指定 --force")
+    if output.exists() and output.is_dir():
+        raise click.UsageError("--output-file 必须是文件路径")
+    output.parent.mkdir(parents=True, exist_ok=True)
+    output.write_text(
+        yaml.safe_dump(template, allow_unicode=True, sort_keys=False),
+        encoding="utf-8",
+    )
+    _render({"template": template_name, "outputFile": str(output)}, output_format)
+
+
+@evalset.command("preview")
+@click.option(
+    "--evalset-file",
+    required=True,
+    type=click.Path(exists=True, dir_okay=False, path_type=Path),
+)
+@click.option(
+    "--data-policy",
+    type=click.Choice(_DATA_POLICIES),
+    default="full_trace",
+    show_default=True,
+)
+@click.option("--format", "output_format", type=click.Choice(["pretty", "json"]), default="pretty")
+def preview(evalset_file: Path, data_policy: str, output_format: str) -> None:
+    """Show the exact fixed-schema payload that a push would publish."""
+    _evalset, snapshot = _load_snapshot(evalset_file, data_policy)
+    _render(snapshot.model_dump(mode="json", by_alias=True, exclude_none=True), output_format)
+
+
+@evalset.command("push")
+@click.option(
+    "--file",
+    "--evalset-file",
+    "evalset_file",
+    required=True,
+    type=click.Path(exists=True, dir_okay=False, path_type=Path),
+)
+@click.option("--dataset-id")
+@click.option("--account-id", envvar="AGENT_EVAL_ACCOUNT_ID", hidden=True)
+@click.option("--idempotency-key", hidden=True)
+@click.option(
+    "--data-policy",
+    type=click.Choice(_DATA_POLICIES),
+    default="full_trace",
+    hidden=True,
+)
+@click.option("--format", "output_format", type=click.Choice(["pretty", "json"]), default="pretty")
+def push(
+    evalset_file: Path,
+    dataset_id: str | None,
+    account_id: str | None,
+    idempotency_key: str | None,
+    data_policy: str,
+    output_format: str,
+) -> None:
+    """Publish a full EvalSet snapshot to the EvalSmith-backed agent-eval API."""
+    workspace = Path.cwd().resolve()
+    try:
+        evalset_path = evalset_file.resolve().relative_to(workspace).as_posix()
+    except ValueError as exc:
+        raise click.UsageError("--file must be inside the current workspace") from exc
+    evalset, _snapshot = _load_snapshot(evalset_file, data_policy)
+    client = AgentEvalCloudDatasetClient(
+        account_id=account_id,
+    )
+    service = CloudEvalSetService(workspace, client)
+    try:
+        result = asyncio.run(
+            service.publish(
+                evalset,
+                evalset_path=evalset_path,
+                dataset_id=dataset_id,
+                data_policy=DataPolicy(data_policy),
+                idempotency_key=idempotency_key,
+            )
+        )
+    except (AgentEvalCloudClientError, CloudEvalSetPreviewError, ValueError) as exc:
+        raise click.ClickException(str(exc)) from exc
+    _render(
+        {
+            "datasetId": result.dataset_id,
+            "datasetVersion": result.dataset_version,
+            "projectId": result.project_id,
+            "schemaHash": result.schema_hash,
+            "contentDigest": result.content_digest,
+            "rowCount": result.row_count,
+        },
+        output_format,
+    )
+
+
+@evalset.command("pull")
+@click.option("--dataset-id", required=True)
+@click.option("--dataset-version", required=True, type=click.IntRange(1))
+@click.option("--project-id")
+@click.option("--output-file", required=True, type=click.Path(dir_okay=False, path_type=Path))
+@click.option("--format", "output_format", type=click.Choice(["pretty", "json"]), default="pretty")
+def pull(
+    dataset_id: str,
+    dataset_version: int,
+    project_id: str | None,
+    output_file: Path,
+    output_format: str,
+) -> None:
+    """Read one immutable cloud Dataset version into a local EvalSet file."""
+    workspace = Path.cwd().resolve()
+    target = output_file.resolve()
+    try:
+        target.relative_to(workspace)
+    except ValueError as exc:
+        raise click.UsageError("--output-file must be inside the current workspace") from exc
+
+    service = CloudEvalSetService(
+        workspace,
+        AgentEvalCloudDatasetClient(),
+    )
+    try:
+        result = asyncio.run(
+            service.pull(
+                dataset_id=dataset_id,
+                version=dataset_version,
+                project_id=project_id,
+            )
+        )
+    except (AgentEvalCloudClientError, CloudEvalSetPreviewError, ValueError) as exc:
+        raise click.ClickException(str(exc)) from exc
+
+    target.parent.mkdir(parents=True, exist_ok=True)
+    target.write_text(
+        yaml.safe_dump(
+            result.evalset.model_dump(mode="json", by_alias=True, exclude_none=True),
+            allow_unicode=True,
+            sort_keys=False,
+        ),
+        encoding="utf-8",
+    )
+    _render(
+        {
+            "outputFile": target.relative_to(workspace).as_posix(),
+            "datasetId": result.cloud_dataset.dataset_id,
+            "datasetVersion": result.cloud_dataset.version,
+            "schemaHash": result.cloud_dataset.schema_hash,
+            "contentDigest": result.cloud_dataset.content_digest,
+            "rowCount": result.cloud_dataset.row_count,
+        },
+        output_format,
+    )

ksadk/cli/cmd_files.py

@@ -607,7 +607,11 @@ def _resolve_workspace_command_context(
     cwd = Path(".").resolve()
     state = load_state(cwd)
     resolved_region = _resolve_workspace_region(region, state)
-    target_agent = _resolve_workspace_agent_ref(agent_input, cwd)
+    # An explicit runtime endpoint is authoritative. Do not leak an unrelated
+    # Agent id from the current workspace state into direct-runtime requests.
+    target_agent = (
+        None if endpoint and not agent_input else _resolve_workspace_agent_ref(agent_input, cwd)
+    )
     resolved_endpoint, resolved_api_key = _resolve_workspace_runtime_access(
         state=state,
         target_agent=target_agent,

ksadk/cli/cmd_hermes.py

@@ -16,7 +16,6 @@ from ksadk.cli.agent_ref import merge_agent_inputs, resolve_agent_ref
 from ksadk.cli.cmd_dashboard import _open_dashboard
 from ksadk.cli.dry_run import dry_run_option, effective_dry_run, run_async_with_dry_run
 from ksadk.cli.env_options import (
-    apply_explicit_env_with_shell_priority,
     env_options,
     inject_env_to_environ,
     resolve_runtime_env_overrides,
@@ -53,18 +52,17 @@ from ksadk.cli.ui import (
 from ksadk.cli.ui import (
     output_option as cli_output_option,
 )
-from ksadk.configs.env_registry import is_sensitive_env_var
 from ksadk.deployment.agent_access import (
     get_latest_agent_access,
     is_agent_not_found_error,
     normalize_deployment_status,
 )
 from ksadk.deployment.state import clear_state, load_state, save_state
+from ksadk.cli.hermes_env import _build_hermes_env_vars
 from ksadk.hermes_terminal import (
     run_hermes_terminal_session,
     validate_hermes_pairing_argv,
 )
-from ksadk.model_policy import build_runtime_model_policy_env
 
 DEFAULT_HERMES_IMAGE = "ghcr.io/kingsoftcloud/hermes-agent:v2026.7.7.2-ksadk-v070"
 DEFAULT_HERMES_CONTEXT_LENGTHS = (("glm-5.1", "200000"),)
@@ -203,22 +201,6 @@ def _env_value(*names: str) -> str:
     return ""
 
 
-def _normalize_hermes_ui_locale(raw: Optional[str]) -> str:
-    """标准化 Hermes UI 语言代码，当前 upstream 只支持 en / zh。"""
-    text = str(raw or "").strip()
-    if not text:
-        return "zh"
-
-    base = text.split(".", 1)[0].replace("_", "-").strip().lower()
-    if base in {"c", "c-utf-8", "c.utf-8", "posix"}:
-        return "zh"
-    if base.startswith("en"):
-        return "en"
-    if base.startswith("zh"):
-        return "zh"
-    return "zh"
-
-
 async def _fetch_hermes_bootstrap_config(region: str) -> dict[str, Any] | None:
     """从服务端获取 Hermes 客户端启动配置。失败时返回 None。"""
     from ksadk.version import VERSION as CLI_VERSION
@@ -252,108 +234,6 @@ def _extract_hermes_bootstrap_image(bootstrap_cfg: dict[str, Any] | None) -> str
     return str(value or "").strip()
 
 
-def _default_context_length_for_model(model: str | None) -> str:
-    normalized = str(model or "").strip().lower()
-    if not normalized:
-        return ""
-    for model_fragment, context_length in DEFAULT_HERMES_CONTEXT_LENGTHS:
-        if model_fragment in normalized:
-            return context_length
-    return ""
-
-
-def _build_hermes_env_vars(
-    *,
-    model_base_url: str | None = None,
-    model_api_key: str | None = None,
-    default_model: str | None = None,
-    model_metadata: dict[str, Any] | None = None,
-    cli_env: dict[str, str] | None = None,
-    auto_dotenv: dict[str, str] | None = None,
-    shell_keys: set[str] | None = None,
-) -> list[dict[str, Any]]:
-    raw_model_base_url = model_base_url or _env_value("OPENAI_BASE_URL")
-    resolved_model_base_url = (
-        _normalize_hermes_runtime_base_url(raw_model_base_url)
-        if raw_model_base_url
-        else DEFAULT_HERMES_RUNTIME_BASE_URL
-    )
-    resolved_default_model = (
-        default_model or _env_value("OPENAI_MODEL_NAME") or DEFAULT_HERMES_MODEL_NAME
-    )
-    metadata_context_length = ""
-    if isinstance(model_metadata, dict):
-        metadata_context_length = str(model_metadata.get("context_window_tokens") or "").strip()
-    context_length = (
-        _env_value("HERMES_CONTEXT_LENGTH", "OPENAI_CONTEXT_LENGTH", "MODEL_CONTEXT_LENGTH")
-        or metadata_context_length
-        or _default_context_length_for_model(resolved_default_model)
-    )
-    ui_locale = _normalize_hermes_ui_locale(_env_value("HERMES_UI_LOCALE", "LANG", "LC_ALL"))
-    raw = {
-        "OPENAI_API_KEY": model_api_key or _env_value("OPENAI_API_KEY"),
-        "OPENAI_BASE_URL": resolved_model_base_url,
-        "OPENAI_MODEL_NAME": resolved_default_model,
-        "API_SERVER_ENABLED": "true",
-        "API_SERVER_HOST": "127.0.0.1",
-        "API_SERVER_PORT": "8642",
-        "HERMES_DASHBOARD_HOST": "127.0.0.1",
-        "HERMES_DASHBOARD_PORT": "9119",
-        "KSADK_RUNTIME_PORT": _env_value("PORT") or "8080",
-        "HERMES_UI_LOCALE": ui_locale,
-    }
-    if context_length:
-        raw["HERMES_CONTEXT_LENGTH"] = context_length
-    fallback_model = _env_value("HERMES_FALLBACK_MODEL", "OPENAI_FALLBACK_MODEL_NAME")
-    if fallback_model:
-        raw["HERMES_FALLBACK_PROVIDER"] = _env_value("HERMES_FALLBACK_PROVIDER") or "custom"
-        raw["HERMES_FALLBACK_MODEL"] = fallback_model
-        raw["HERMES_FALLBACK_BASE_URL"] = (
-            _env_value("HERMES_FALLBACK_BASE_URL") or resolved_model_base_url
-        )
-    api_server_key = _env_value("API_SERVER_KEY", "HERMES_API_SERVER_KEY")
-    if api_server_key:
-        raw["API_SERVER_KEY"] = api_server_key
-    # Observability routes and credentials are platform-managed. The Hermes
-    # deploy CLI must not translate or forward legacy Langfuse SDK variables;
-    # server/runtime inject the standard OTLP primary and CloudMonitor secondary.
-    for key in (
-        "WPSXIEZUO_APP_ID",
-        "WPSXIEZUO_APP_KEY",
-        "WPSXIEZUO_API_BASE",
-        "WPSXIEZUO_WS_ENDPOINT",
-        "WPSXIEZUO_GROUP_AT_ONLY",
-        "WPSXIEZUO_ALLOWED_USERS",
-        "WPSXIEZUO_ALLOW_ALL_USERS",
-        "WPSXIEZUO_HOME_CHANNEL",
-    ):
-        value = _env_value(key)
-        if value:
-            raw[key] = value
-    raw = build_runtime_model_policy_env(raw, runtime="hermes")
-    if raw.get("HERMES_FALLBACK_MODEL"):
-        raw.setdefault(
-            "HERMES_FALLBACK_PROVIDER", _env_value("HERMES_FALLBACK_PROVIDER") or "custom"
-        )
-        raw.setdefault(
-            "HERMES_FALLBACK_BASE_URL",
-            _env_value("HERMES_FALLBACK_BASE_URL") or resolved_model_base_url,
-        )
-    if cli_env or auto_dotenv:
-        apply_explicit_env_with_shell_priority(
-            raw, cli_env or {}, auto_dotenv or {}, shell_keys or set(os.environ)
-        )
-    return [
-        {
-            "Key": key,
-            "Value": str(value),
-            "IsSensitive": is_sensitive_env_var(key),
-        }
-        for key, value in raw.items()
-        if value is not None and str(value).strip() != ""
-    ]
-
-
 def _validate_hermes_model_config(
     *,
     model_base_url: str | None = None,
@@ -371,11 +251,6 @@ def _validate_hermes_model_config(
         print_info("未配置 OPENAI_API_KEY，将由服务端在需要时自动创建。")
 
 
-def _normalize_hermes_runtime_base_url(base_url: str | None) -> str:
-    normalized = str(base_url or "").strip()
-    return normalized
-
-
 _FAILURE_STATUSES = {"FAILED", "ERROR", "TERMINATED"}