agentengine-sdk-python 0.7.0__py3-none-any.whl → 0.8.1__py3-none-any.whl

ksadk/studio/evaluation.py

@@ -0,0 +1,192 @@
+"""Local evaluation suites and deterministic assertions."""
+
+from __future__ import annotations
+
+import json
+from collections.abc import Awaitable, Callable
+from pathlib import Path
+from typing import cast
+from uuid import uuid4
+
+from jsonschema import (  # type: ignore[import-untyped]
+    ValidationError as JSONSchemaValidationError,
+)
+from jsonschema import validate as validate_json  # type: ignore[import-untyped]
+
+from ksadk.studio.contracts import (
+    AssertionResult,
+    AssertionSpec,
+    EvaluationCaseResult,
+    EvaluationRun,
+    EvaluationSuite,
+    RunRecord,
+    RunStatus,
+)
+from ksadk.studio.errors import StudioError
+from ksadk.studio.event_store import RunEventStore
+from ksadk.studio.repository import BuildRepository, load_yaml_file
+from ksadk.studio.workspace import Workspace
+
+
+class EvaluationRunner:
+    def __init__(
+        self,
+        workspace: Workspace,
+        *,
+        run_agent: Callable[[str, str, str | None], Awaitable[RunRecord]],
+        event_store: RunEventStore,
+        build_repository: BuildRepository | None = None,
+    ) -> None:
+        self.workspace = workspace
+        self.run_agent = run_agent
+        self.event_store = event_store
+        self.build_repository = build_repository or BuildRepository(workspace)
+
+    async def run(
+        self,
+        build_id: str,
+        suite_refs: list[str],
+        *,
+        fail_fast: bool = False,
+    ) -> EvaluationRun:
+        build = self.build_repository.get(build_id)
+        suites = [
+            self._load_suite(build.agent_id, reference) for reference in suite_refs
+        ]
+        evaluation = EvaluationRun(
+            id=f"eval_{uuid4().hex}",
+            build_id=build_id,
+            status=RunStatus.RUNNING,
+        )
+        for suite in suites:
+            for case in suite.cases:
+                run = await self.run_agent(
+                    build_id,
+                    case.input,
+                    f"eval_{evaluation.id}_{case.id}",
+                )
+                assertion_results = [
+                    self._assert(assertion, run) for assertion in case.assertions
+                ]
+                passed = run.status == RunStatus.COMPLETED and all(
+                    result.passed for result in assertion_results
+                )
+                evaluation.results.append(
+                    EvaluationCaseResult(
+                        case_id=case.id,
+                        run_id=run.id,
+                        passed=passed,
+                        assertions=assertion_results,
+                    )
+                )
+                if fail_fast and not passed:
+                    break
+            if fail_fast and evaluation.results and not evaluation.results[-1].passed:
+                break
+        evaluation.total = len(evaluation.results)
+        evaluation.passed = sum(1 for result in evaluation.results if result.passed)
+        evaluation.failed = evaluation.total - evaluation.passed
+        evaluation.pass_rate = (
+            evaluation.passed / evaluation.total if evaluation.total else 0
+        )
+        evaluation.status = (
+            RunStatus.COMPLETED if evaluation.failed == 0 else RunStatus.FAILED
+        )
+        self._save(evaluation)
+        return evaluation
+
+    def get(self, evaluation_id: str) -> EvaluationRun:
+        path = self.workspace.resolve(
+            Path(".agentkit/evaluations") / f"{evaluation_id}.json"
+        )
+        if not path.is_file():
+            raise StudioError(
+                "EVALUATION_NOT_FOUND",
+                "Evaluation 不存在",
+                status_code=404,
+                details={"id": evaluation_id},
+            )
+        return cast(
+            EvaluationRun,
+            EvaluationRun.model_validate_json(path.read_text(encoding="utf-8")),
+        )
+
+    def _load_suite(self, agent_id: str, reference: str) -> EvaluationSuite:
+        candidates = [
+            Path("agents") / agent_id / reference,
+            Path(reference),
+        ]
+        for candidate in candidates:
+            path = self.workspace.resolve(candidate)
+            if path.is_file():
+                try:
+                    return cast(
+                        EvaluationSuite,
+                        EvaluationSuite.model_validate(load_yaml_file(path)),
+                    )
+                except ValueError as exc:
+                    raise StudioError(
+                        "EVALUATION_SUITE_INVALID",
+                        "评测集格式无效",
+                        status_code=422,
+                        details={"reference": reference, "reason": str(exc)},
+                    ) from exc
+        raise StudioError(
+            "EVALUATION_SUITE_NOT_FOUND",
+            "评测集不存在",
+            status_code=404,
+            details={"reference": reference},
+        )
+
+    def _assert(self, assertion: AssertionSpec, run: RunRecord) -> AssertionResult:
+        value = assertion.value
+        passed = False
+        message = ""
+        if assertion.type == "contains":
+            passed = str(value) in run.output
+        elif assertion.type == "equals":
+            passed = run.output == str(value)
+        elif assertion.type == "notContains":
+            passed = str(value) not in run.output
+        elif assertion.type == "maxLatencyMs":
+            passed = (run.duration_ms or 0) <= int(value)
+        elif assertion.type == "maxInputTokens":
+            passed = run.usage.input_tokens <= int(value)
+        elif assertion.type == "maxOutputTokens":
+            passed = run.usage.output_tokens <= int(value)
+        elif assertion.type == "jsonSchema":
+            try:
+                validate_json(json.loads(run.output), value)
+                passed = True
+            except (ValueError, JSONSchemaValidationError) as exc:
+                message = str(exc)
+        elif assertion.type in {"toolCalled", "toolNotCalled"}:
+            called = {
+                event.data.get("tool")
+                for event in self.event_store.events(run.id)
+                if event.type == "tool.requested"
+            }
+            passed = (
+                str(value) in called
+                if assertion.type == "toolCalled"
+                else str(value) not in called
+            )
+        if not message and not passed:
+            message = f"断言 {assertion.type} 未通过"
+        return AssertionResult(assertion=assertion, passed=passed, message=message)
+
+    def _save(self, evaluation: EvaluationRun) -> None:
+        directory = self.workspace.resolve(".agentkit/evaluations")
+        directory.mkdir(parents=True, exist_ok=True)
+        self.workspace.atomic_write_text(
+            directory / f"{evaluation.id}.json",
+            json.dumps(
+                evaluation.model_dump(
+                    by_alias=True, exclude_none=True, mode="json"
+                ),
+                ensure_ascii=False,
+                sort_keys=True,
+                indent=2,
+            )
+            + "\n",
+        )

ksadk/studio/event_store.py

@@ -0,0 +1,284 @@
+"""Persistent local Run and Event store."""
+
+from __future__ import annotations
+
+import json
+import shutil
+from datetime import datetime, timezone
+from pathlib import Path
+from typing import List
+
+from pydantic import ValidationError
+
+from ksadk.studio.contracts import RunEvent, RunRecord, RunStatus
+from ksadk.studio.errors import StudioError, not_found
+from ksadk.studio.otel_trace import OtlpTraceStore
+from ksadk.studio.workspace import Workspace
+
+
+class RunEventStore:
+    def __init__(self, workspace: Workspace) -> None:
+        self.workspace = workspace
+        self.trace_store = OtlpTraceStore(workspace)
+
+    def _path(self, run_id: str) -> Path:
+        return self.workspace.resolve(Path(".agentkit/runs") / f"{run_id}.json")
+
+    def create(self, record: RunRecord) -> RunRecord:
+        path = self._path(record.id)
+        if path.exists():
+            raise StudioError("RUN_ALREADY_EXISTS", "Run 已存在", status_code=409)
+        self._write(record, [])
+        return record
+
+    def save(self, record: RunRecord) -> RunRecord:
+        _, events = self._read(record.id)
+        self._write(record, events)
+        return record
+
+    def append(self, run_id: str, event_type: str, data: dict) -> RunEvent:
+        record, events = self._read(run_id)
+        event = RunEvent(
+            id=len(events) + 1,
+            run_id=run_id,
+            type=event_type,
+            data=data,
+        )
+        events.append(event)
+        self._write(record, events)
+        return event
+
+    def get(self, run_id: str) -> RunRecord:
+        record, _ = self._read(run_id)
+        return record
+
+    def events(self, run_id: str, *, after: int = 0) -> list[RunEvent]:
+        _, events = self._read(run_id)
+        return [event for event in events if event.id > after]
+
+    def list_runs(
+        self,
+        *,
+        session_id: str | None = None,
+        agent_id: str | None = None,
+    ) -> List[RunRecord]:
+        records: list[RunRecord] = []
+        directory = self.workspace.resolve(".agentkit/runs")
+        for path in sorted(directory.glob("run_*.json")):
+            try:
+                payload = json.loads(path.read_text(encoding="utf-8"))
+                record = RunRecord.model_validate(payload["record"])
+            except (OSError, ValueError, KeyError, ValidationError):
+                continue
+            if session_id and record.session_id != session_id:
+                continue
+            if agent_id and record.agent_id != agent_id:
+                continue
+            records.append(record)
+        records.sort(
+            key=lambda item: (
+                item.started_at or item.completed_at or datetime.min.replace(tzinfo=timezone.utc)
+            ),
+            reverse=False,
+        )
+        return records
+
+    def delete_session(self, session_id: str) -> int:
+        """Delete every persisted run that belongs to one local chat session."""
+
+        deleted = 0
+        for record in self.list_runs(session_id=session_id):
+            path = self._path(record.id)
+            if path.is_file():
+                self.trace_store.delete(record.trace_id, purge=True)
+                path.unlink()
+                deleted += 1
+        return deleted
+
+    def delete_agent(
+        self,
+        agent_id: str,
+        *,
+        purge: bool,
+        trash_directory: Path | None = None,
+    ) -> int:
+        """Remove the persisted Runs and Trace events owned by one Agent."""
+
+        deleted = 0
+        for record in self.list_runs(agent_id=agent_id):
+            path = self._path(record.id)
+            if not path.is_file():
+                continue
+            if purge:
+                self.trace_store.delete(record.trace_id, purge=True)
+                path.unlink()
+            else:
+                if trash_directory is None:
+                    raise ValueError("recoverable deletion requires a trash directory")
+                destination = self.workspace.resolve(trash_directory / "runs" / path.name)
+                destination.parent.mkdir(parents=True, exist_ok=True)
+                self.trace_store.delete(
+                    record.trace_id,
+                    purge=False,
+                    trash_directory=trash_directory,
+                )
+                shutil.move(str(path), str(destination))
+            deleted += 1
+        return deleted
+
+    def recover_interrupted(self) -> int:
+        """Reconcile non-terminal records left behind by a stopped Studio.
+
+        Events are persisted before the final RunRecord update.  A browser
+        disconnect or process stop can therefore leave a terminal event next
+        to a stale ``RUNNING`` record.  On startup the event log wins; a run
+        without any terminal event is explicitly marked interrupted because
+        its in-memory runtime task cannot survive a Studio restart.
+        """
+
+        recovered = 0
+        for record in self.list_runs():
+            if record.status not in {
+                RunStatus.CREATED,
+                RunStatus.RUNNING,
+                RunStatus.PAUSED,
+                RunStatus.WAITING_INPUT,
+            }:
+                continue
+            events = self.events(record.id)
+            terminal = next(
+                (
+                    event
+                    for event in reversed(events)
+                    if event.type
+                    in {
+                        "run.completed",
+                        "run.failed",
+                        "run.cancelled",
+                        "run.interrupted",
+                    }
+                ),
+                None,
+            )
+            if terminal is None:
+                terminal = self.append(
+                    record.id,
+                    "run.interrupted",
+                    {
+                        "status": "interrupted",
+                        "reason": "studio_restarted",
+                    },
+                )
+
+            if terminal.type == "run.completed":
+                record.status = RunStatus.COMPLETED
+                record.error = None
+            elif terminal.type == "run.failed":
+                record.status = RunStatus.FAILED
+                record.error = {
+                    "code": "RUNTIME_RUN_FAILED",
+                    "message": str(
+                        terminal.data.get("error")
+                        or terminal.data.get("message")
+                        or "Agent 运行失败"
+                    ),
+                }
+            elif terminal.type == "run.cancelled":
+                record.status = RunStatus.CANCELLED
+                record.error = {"code": "RUN_CANCELLED", "message": "运行已取消"}
+            else:
+                record.status = RunStatus.INTERRUPTED
+                record.error = {
+                    "code": "RUN_INTERRUPTED",
+                    "message": "Studio 重启后无法重新 attach 上一次本地运行",
+                }
+            record.completed_at = terminal.created_at
+            if record.started_at is not None:
+                record.duration_ms = max(
+                    0,
+                    int((record.completed_at - record.started_at).total_seconds() * 1000),
+                )
+            self.save(record)
+            recovered += 1
+        return recovered
+
+    def trace(self, trace_id: str) -> dict:
+        return self.trace_store.get_trace_view(trace_id)
+
+    def trace_otlp(self, trace_id: str) -> dict:
+        return self.trace_store.get_otlp(trace_id)
+
+    def list_traces(
+        self,
+        *,
+        agent_id: str | None = None,
+        status: str | None = None,
+        limit: int = 200,
+    ) -> list[dict]:
+        return self.trace_store.list_trace_summaries(
+            agent_id=agent_id,
+            status=status,
+            limit=limit,
+        )
+
+    def list_traces_page(
+        self,
+        *,
+        agent_id: str | None = None,
+        status: str | None = None,
+        query: str = "",
+        limit: int = 50,
+        cursor: str | None = None,
+        sort: str = "startedAt:desc",
+    ) -> dict:
+        return self.trace_store.paginate_trace_summaries(
+            agent_id=agent_id,
+            status=status,
+            query=query,
+            limit=limit,
+            cursor=cursor,
+            sort=sort,
+        )
+
+    def trace_overview(
+        self,
+        *,
+        range_name: str = "24h",
+        agent_id: str | None = None,
+        status: str | None = None,
+    ) -> dict:
+        return self.trace_store.trace_overview(
+            range_name=range_name,
+            agent_id=agent_id,
+            status=status,
+        )
+
+    def _read(self, run_id: str) -> tuple[RunRecord, List[RunEvent]]:
+        path = self._path(run_id)
+        if not path.is_file():
+            raise not_found("run", run_id)
+        try:
+            payload = json.loads(path.read_text(encoding="utf-8"))
+            record = RunRecord.model_validate(payload["record"])
+            events = [RunEvent.model_validate(item) for item in payload["events"]]
+            return record, events
+        except (OSError, ValueError, KeyError, ValidationError) as exc:
+            raise StudioError(
+                "RUN_RECORD_INVALID",
+                "Run 记录损坏",
+                status_code=500,
+                details={"id": run_id},
+            ) from exc
+
+    def _write(self, record: RunRecord, events: List[RunEvent]) -> None:
+        payload = {
+            "record": record.model_dump(by_alias=True, exclude_none=True, mode="json"),
+            "events": [
+                event.model_dump(by_alias=True, exclude_none=True, mode="json") for event in events
+            ],
+        }
+        self.workspace.atomic_write_text(
+            self._path(record.id),
+            json.dumps(payload, ensure_ascii=False, sort_keys=True, indent=2) + "\n",
+        )
+        self.trace_store.sync(record, events)

ksadk/studio/framework_run.py

@@ -0,0 +1,114 @@
+"""Resolve immutable ADK/LangGraph Studio builds into RuntimeLaunchContext."""
+
+from __future__ import annotations
+
+import json
+
+from ksadk.detection.detector import FrameworkDetector
+from ksadk.runtime import RuntimeLaunchContext
+from ksadk.studio.errors import StudioError
+from ksadk.studio.repository import BuildRepository
+from ksadk.studio.run_service import StudioRunSpec
+from ksadk.studio.workspace import Workspace
+from ksadk.tools.gateway import normalize_tool_approval_mode
+
+
+class FrameworkRunSpecResolver:
+    def __init__(
+        self,
+        workspace: Workspace,
+        *,
+        build_repository: BuildRepository | None = None,
+    ) -> None:
+        self.workspace = workspace
+        self.builds = build_repository or BuildRepository(workspace)
+
+    def resolve(
+        self,
+        build_id: str,
+        *,
+        model: str | None = None,
+        approval_mode: str | None = None,
+    ) -> StudioRunSpec:
+        build = self.builds.get(build_id)
+        runtime_type = build.runtime_type.strip().lower()
+        if runtime_type not in {"adk", "langgraph"}:
+            raise StudioError(
+                "BUILD_RUNTIME_UNSUPPORTED",
+                "Build 没有可由 RuntimeAdapter 启动的 ADK/LangGraph Runtime",
+                status_code=422,
+                details={"buildId": build_id, "runtimeType": runtime_type},
+            )
+        if not build.artifact_path:
+            raise StudioError("BUILD_NOT_READY", "Build 尚未生成制品", status_code=409)
+        artifact_root = self.workspace.resolve(build.artifact_path, must_exist=True).parent
+        bundle_root = artifact_root / "agent-bundle"
+        project_dir = bundle_root / "runtime"
+        if not project_dir.is_dir():
+            raise StudioError(
+                "BUILD_RUNTIME_SOURCE_MISSING",
+                "Build 缺少不可变 Runtime 源码快照",
+                status_code=500,
+            )
+        detection = FrameworkDetector(str(project_dir)).detect()
+        if not detection.is_valid or detection.type.value != runtime_type:
+            raise StudioError(
+                "BUILD_RUNTIME_DETECTION_MISMATCH",
+                "Build 中的 Runtime 源码与 Runtime Lock 不一致",
+                status_code=422,
+                details={
+                    "expected": runtime_type,
+                    "detected": detection.type.value,
+                },
+            )
+        selected_model = self._select_model(build.runtime_lock, model)
+        resolved_path = bundle_root / "resolved-agent-spec.json"
+        resolved = json.loads(resolved_path.read_text(encoding="utf-8"))
+        instructions = resolved.get("instructions") if isinstance(resolved, dict) else {}
+        request_config = {
+            "base_instructions": str((instructions or {}).get("system") or ""),
+            "entry_point": detection.entry_point,
+            "agent_variable": detection.agent_variable,
+        }
+        if approval_mode:
+            request_config["tool_approval_mode"] = normalize_tool_approval_mode(
+                approval_mode
+            )
+        return StudioRunSpec(
+            launch_context=RuntimeLaunchContext(
+                runtime_type=runtime_type,
+                project_dir=project_dir,
+                detection=detection,
+                config=dict(detection.raw_config or {}),
+            ),
+            build_id=build.id,
+            agent_id=build.agent_id,
+            model=selected_model,
+            request_config=request_config,
+            manifest_sha256=build.resolved_digest,
+        )
+
+    @staticmethod
+    def _select_model(runtime_lock: dict, requested: str | None) -> str:
+        allowed = [str(value) for value in runtime_lock.get("models") or [] if str(value)]
+        default = str(runtime_lock.get("model") or "").strip()
+        if default and default not in allowed:
+            allowed.insert(0, default)
+        selected = str(requested or default).strip()
+        if not selected:
+            raise StudioError(
+                "AGENT_MODEL_REQUIRED",
+                "Build 没有绑定可运行模型",
+                status_code=422,
+            )
+        if allowed and selected not in allowed:
+            raise StudioError(
+                "MODEL_NOT_BOUND",
+                "请求模型未绑定到当前 Agent Build",
+                status_code=422,
+                details={"model": selected, "allowedModels": allowed},
+            )
+        return selected
+
+
+__all__ = ["FrameworkRunSpecResolver"]

ksadk/studio/identifiers.py

@@ -0,0 +1,33 @@
+"""Server-owned local identifiers for Studio resources."""
+
+from __future__ import annotations
+
+import re
+import secrets
+from collections.abc import Callable
+
+from ksadk.studio.errors import StudioError
+
+_GENERATED_AGENT_SLUG = re.compile(r"^agentkit-[0-9a-f]{8}$")
+
+
+def generate_agent_slug(exists: Callable[[str], bool]) -> str:
+    """Allocate a short opaque Agent ID without trusting a client-side default."""
+
+    for _attempt in range(32):
+        candidate = f"agentkit-{secrets.token_hex(4)}"
+        if not exists(candidate):
+            return candidate
+    raise StudioError(
+        "AGENT_ID_EXHAUSTED",
+        "无法分配唯一的本地标识",
+        status_code=503,
+        field="slug",
+    )
+
+
+def is_generated_agent_slug(value: str) -> bool:
+    return bool(_GENERATED_AGENT_SLUG.fullmatch(str(value or "").strip()))
+
+
+__all__ = ["generate_agent_slug", "is_generated_agent_slug"]