agentengine-sdk-python 0.7.0__py3-none-any.whl → 0.8.1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {agentengine_sdk_python-0.7.0.dist-info → agentengine_sdk_python-0.8.1.dist-info}/METADATA +52 -20
- agentengine_sdk_python-0.8.1.dist-info/RECORD +644 -0
- {agentengine_sdk_python-0.7.0.dist-info → agentengine_sdk_python-0.8.1.dist-info}/WHEEL +1 -1
- ksadk/a2a/__init__.py +180 -11
- ksadk/a2a/bootstrap.py +332 -0
- ksadk/a2a/card.py +15 -0
- ksadk/a2a/context_store.py +203 -0
- ksadk/a2a/control_plane.py +1056 -0
- ksadk/a2a/credential.py +99 -0
- ksadk/a2a/event_adapter.py +149 -0
- ksadk/a2a/executor.py +361 -137
- ksadk/a2a/external_transport.py +516 -0
- ksadk/a2a/identity.py +304 -0
- ksadk/a2a/ids.py +22 -0
- ksadk/a2a/langgraph.py +285 -0
- ksadk/a2a/resume_store.py +201 -0
- ksadk/a2a/routes.py +112 -0
- ksadk/a2a/server.py +102 -62
- ksadk/a2a/service_env.py +33 -0
- ksadk/a2a/space_client.py +1122 -0
- ksadk/a2a/task_adapter.py +423 -0
- ksadk/a2a/task_event_dispatcher.py +167 -0
- ksadk/a2a/task_event_outbox.py +326 -0
- ksadk/a2a/task_store.py +206 -0
- ksadk/a2a_card.py +101 -0
- ksadk/a2ui/__init__.py +27 -0
- ksadk/a2ui/core.py +186 -0
- ksadk/a2ui/models.py +166 -0
- ksadk/a2ui/renderer.py +198 -0
- ksadk/agui/__init__.py +9 -0
- ksadk/agui/a2ui_projection.py +92 -0
- ksadk/agui/agent.py +872 -0
- ksadk/agui/config.py +92 -0
- ksadk/agui/routes.py +47 -0
- ksadk/api/client.py +238 -204
- ksadk/api/environment.py +27 -0
- ksadk/builders/__init__.py +2 -0
- ksadk/builders/base.py +16 -14
- ksadk/builders/code_builder.py +375 -270
- ksadk/builders/container_builder.py +240 -200
- ksadk/builders/framework_requirements.py +31 -12
- ksadk/builders/ks3_uploader.py +60 -29
- ksadk/builders/managed_runtime_builder.py +173 -0
- ksadk/builders/mcp_builder.py +91 -92
- ksadk/builders/requirements_utils.py +1 -5
- ksadk/cli/__init__.py +97 -171
- ksadk/cli/cmd_a2a.py +255 -84
- ksadk/cli/cmd_agent.py +19 -5
- ksadk/cli/cmd_build.py +158 -29
- ksadk/cli/cmd_completion.py +47 -35
- ksadk/cli/cmd_config.py +267 -198
- ksadk/cli/cmd_create.py +507 -740
- ksadk/cli/cmd_dashboard.py +120 -45
- ksadk/cli/cmd_deploy.py +135 -59
- ksadk/cli/cmd_destroy.py +44 -14
- ksadk/cli/cmd_eval.py +386 -0
- ksadk/cli/cmd_files.py +63 -24
- ksadk/cli/cmd_hermes.py +313 -135
- ksadk/cli/cmd_invoke.py +184 -122
- ksadk/cli/cmd_launch.py +76 -28
- ksadk/cli/cmd_mcp.py +98 -64
- ksadk/cli/cmd_model.py +14 -5
- ksadk/cli/cmd_openclaw.py +564 -268
- ksadk/cli/cmd_replay.py +71 -0
- ksadk/cli/cmd_run.py +58 -73
- ksadk/cli/cmd_status.py +80 -31
- ksadk/cli/cmd_studio.py +110 -0
- ksadk/cli/cmd_version.py +20 -13
- ksadk/cli/cmd_web.py +95 -18
- ksadk/cli/deploy_utils.py +20 -20
- ksadk/cli/dry_run.py +7 -4
- ksadk/cli/env_options.py +83 -0
- ksadk/cli/error_utils.py +37 -12
- ksadk/cli/global_options.py +4 -3
- ksadk/cli/local_runtime.py +1 -2
- ksadk/cli/model_catalog.py +6 -2
- ksadk/cli/network_options.py +41 -19
- ksadk/cli/resource_common.py +6 -2
- ksadk/cli/runtime_bootstrap.py +51 -0
- ksadk/cli/storage.py +0 -1
- ksadk/cli/ui.py +11 -3
- ksadk/cli/workflow_common.py +52 -18
- ksadk/codex/__init__.py +12 -0
- ksadk/codex/client.py +1042 -0
- ksadk/codex/phase.py +143 -0
- ksadk/codex/runtime.py +940 -0
- ksadk/common/aicp_env.py +0 -1
- ksadk/common/auth.py +2 -2
- ksadk/common/constants.py +1 -3
- ksadk/common/kop_client.py +232 -0
- ksadk/common/llm_utils.py +17 -13
- ksadk/compat/__init__.py +5 -0
- ksadk/compat/adk_compat.py +200 -0
- ksadk/compat/copilotkit_a2ui.py +330 -0
- ksadk/configs/__init__.py +12 -14
- ksadk/configs/env_registry.py +661 -116
- ksadk/configs/global_config.py +3 -2
- ksadk/configs/settings.py +133 -209
- ksadk/conversations/__init__.py +85 -92
- ksadk/conversations/attachment_storage.py +16 -11
- ksadk/conversations/attachments.py +44 -36
- ksadk/conversations/compaction_pipeline.py +12 -12
- ksadk/conversations/compaction_prompt.py +9 -5
- ksadk/conversations/context.py +221 -6
- ksadk/conversations/message_projection.py +438 -51
- ksadk/conversations/model_context.py +12 -6
- ksadk/conversations/model_options.py +0 -1
- ksadk/conversations/normalize.py +20 -16
- ksadk/conversations/reasoning_markup.py +1 -1
- ksadk/conversations/runtime.py +339 -5255
- ksadk/conversations/runtime_compaction.py +333 -0
- ksadk/conversations/runtime_constants.py +27 -0
- ksadk/conversations/runtime_governance.py +185 -0
- ksadk/conversations/runtime_input.py +660 -0
- ksadk/conversations/runtime_invocation.py +395 -0
- ksadk/conversations/runtime_metadata.py +475 -0
- ksadk/conversations/runtime_observability.py +367 -0
- ksadk/conversations/runtime_payloads.py +272 -0
- ksadk/conversations/runtime_persistence.py +440 -0
- ksadk/conversations/runtime_preparation.py +426 -0
- ksadk/conversations/runtime_resume.py +836 -0
- ksadk/conversations/runtime_stream_events.py +902 -0
- ksadk/conversations/runtime_streaming.py +700 -0
- ksadk/conversations/semantic_summary.py +17 -5
- ksadk/conversations/session_title.py +6 -3
- ksadk/deployment/__init__.py +15 -16
- ksadk/deployment/agent_access.py +12 -23
- ksadk/deployment/base.py +51 -38
- ksadk/deployment/managed_runtime.py +33 -0
- ksadk/deployment/manager.py +85 -71
- ksadk/deployment/providers/serverless.py +379 -210
- ksadk/deployment/registry.py +38 -32
- ksadk/deployment/state.py +11 -11
- ksadk/deployment/ui_config.py +1 -1
- ksadk/detection/__init__.py +1 -1
- ksadk/detection/detector.py +146 -83
- ksadk/detection/mcp_detector.py +9 -6
- ksadk/evaluation/__init__.py +84 -0
- ksadk/evaluation/a2a_adapter.py +444 -0
- ksadk/evaluation/adapters.py +54 -0
- ksadk/evaluation/contracts.py +409 -0
- ksadk/evaluation/evalset.py +351 -0
- ksadk/evaluation/evaluators.py +461 -0
- ksadk/evaluation/executor.py +118 -0
- ksadk/evaluation/storage.py +68 -0
- ksadk/evaluation/target.py +73 -0
- ksadk/events/__init__.py +25 -0
- ksadk/events/parser.py +191 -0
- ksadk/events/replay.py +37 -0
- ksadk/events/runtime_event.py +273 -0
- ksadk/events/store.py +317 -0
- ksadk/harness/__init__.py +35 -0
- ksadk/harness/app.py +269 -0
- ksadk/harness/config.py +181 -0
- ksadk/harness/reasoner.py +105 -0
- ksadk/harness/runtime.py +364 -0
- ksadk/harness/sandbox.py +141 -0
- ksadk/harness/tools.py +222 -0
- ksadk/hermes_terminal.py +28 -12
- ksadk/identity/resolver.py +15 -9
- ksadk/ids.py +32 -0
- ksadk/knowledge_base/adk_tool.py +1 -1
- ksadk/knowledge_base/client.py +38 -33
- ksadk/managed_a2a_card.py +160 -0
- ksadk/managed_runtime.py +195 -0
- ksadk/markdown.py +5 -2
- ksadk/mcp_runtime/__init__.py +9 -6
- ksadk/memory/__init__.py +1 -1
- ksadk/memory/adk/__init__.py +2 -0
- ksadk/memory/adk/backends/base_ltm_backend.py +2 -6
- ksadk/memory/adk/backends/http_ltm_backend.py +13 -21
- ksadk/memory/adk/backends/inmemory_ltm_backend.py +7 -13
- ksadk/memory/adk/backends/sdk_ltm_backend.py +36 -36
- ksadk/memory/adk/long_term_memory.py +34 -37
- ksadk/memory/adk/resilient_session_service.py +4 -5
- ksadk/memory/adk/short_term_memory.py +12 -21
- ksadk/memory/adk_tool.py +3 -2
- ksadk/memory/backends/base.py +19 -19
- ksadk/memory/backends/memory.py +24 -25
- ksadk/memory/backends/redis.py +31 -30
- ksadk/memory/langchain_tool.py +2 -1
- ksadk/memory/manager.py +52 -47
- ksadk/memory/service.py +16 -6
- ksadk/memory/tool.py +4 -0
- ksadk/model_policy.py +15 -7
- ksadk/model_proxy/__init__.py +51 -0
- ksadk/model_proxy/bootstrap.py +80 -0
- ksadk/model_proxy/cache.py +165 -0
- ksadk/model_proxy/config.py +66 -0
- ksadk/model_proxy/demo_webui.py +115 -0
- ksadk/model_proxy/detect.py +145 -0
- ksadk/model_proxy/gate.py +68 -0
- ksadk/model_proxy/namespace.py +201 -0
- ksadk/model_proxy/protocol_proxy.py +22 -0
- ksadk/model_proxy/server.py +443 -0
- ksadk/model_proxy/transform.py +735 -0
- ksadk/openclaw_gateway.py +29 -14
- ksadk/runners/adk_runner.py +354 -146
- ksadk/runners/base_runner.py +71 -29
- ksadk/runners/deepagents_runner.py +0 -1
- ksadk/runners/factory.py +18 -5
- ksadk/runners/langchain_runner.py +29 -742
- ksadk/runners/langgraph_runner.py +417 -97
- ksadk/runners/patch_langchain.py +16 -16
- ksadk/runners/remote_runner.py +134 -33
- ksadk/runners/usage_accumulator.py +1 -0
- ksadk/runners/utils/__init__.py +0 -10
- ksadk/runners/utils/loader.py +23 -7
- ksadk/runtime/__init__.py +67 -0
- ksadk/runtime/adapter.py +412 -0
- ksadk/runtime/conversation_execution.py +553 -0
- ksadk/runtime/executor.py +251 -0
- ksadk/runtime/factory.py +156 -0
- ksadk/runtime/framework_adapters.py +118 -0
- ksadk/runtime/launch.py +48 -0
- ksadk/runtime/preprocessing.py +131 -0
- ksadk/runtime/runner_adapter.py +1032 -0
- ksadk/runtime/runner_loading.py +36 -0
- ksadk/runtime_context.py +18 -3
- ksadk/runtime_state.py +0 -1
- ksadk/sandbox/__init__.py +4 -1
- ksadk/sandbox/backends/__init__.py +4 -1
- ksadk/sandbox/backends/e2b.py +13 -5
- ksadk/sandbox/backends/local_process.py +11 -3
- ksadk/sandbox/base.py +10 -15
- ksadk/sandbox/factory.py +14 -4
- ksadk/sandbox/registry.py +57 -6
- ksadk/server/__init__.py +39 -4
- ksadk/server/api_models.py +15 -5
- ksadk/server/app.py +24 -3928
- ksadk/server/composition.py +47 -0
- ksadk/server/factory.py +622 -0
- ksadk/server/routes/__init__.py +1 -0
- ksadk/server/routes/checkpoint_resolution.py +116 -0
- ksadk/server/routes/common.py +682 -0
- ksadk/server/routes/control.py +331 -0
- ksadk/server/routes/dependencies.py +98 -0
- ksadk/server/routes/misc.py +425 -0
- ksadk/server/routes/models.py +451 -0
- ksadk/server/routes/openai_compat.py +229 -0
- ksadk/server/routes/projection.py +848 -0
- ksadk/server/routes/routers.py +52 -0
- ksadk/server/routes/run.py +519 -0
- ksadk/server/routes/sessions.py +644 -0
- ksadk/server/routes/streaming.py +265 -0
- ksadk/server/routes/workspace.py +414 -0
- ksadk/server/static/assets/{ArtifactsPanel-D-SSn2ee.js → ArtifactsPanel-CakCqxEN.js} +1 -1
- ksadk/server/static/assets/CodeBlock-CynLknyO.js +9 -0
- ksadk/server/static/assets/MathMessageMarkdown-3sQdOysS.js +8 -0
- ksadk/server/static/assets/MermaidBlock-NERD_ef7.js +303 -0
- ksadk/server/static/assets/NativeTerminalPanel-CiHWf82q.js +1 -0
- ksadk/server/static/assets/{arc-BA13sHc9.js → arc-DrgjcKhl.js} +1 -1
- ksadk/server/static/assets/architecture-7EHR7CIX-CpZLRCcY.js +1 -0
- ksadk/server/static/assets/{architectureDiagram-3BPJPVTR-DzJzgVoL.js → architectureDiagram-3BPJPVTR-BXoVRbQ6.js} +1 -1
- ksadk/server/static/assets/{blockDiagram-GPEHLZMM-UTdVcZhn.js → blockDiagram-GPEHLZMM-BnaB3Ywv.js} +1 -1
- ksadk/server/static/assets/{c4Diagram-AAUBKEIU-Dix0GD4m.js → c4Diagram-AAUBKEIU-Dw4weZcp.js} +1 -1
- ksadk/server/static/assets/channel-Dt3rQTFI.js +1 -0
- ksadk/server/static/assets/{chunk-2J33WTMH-C2JxuZ42.js → chunk-2J33WTMH-ChjpLK7j.js} +1 -1
- ksadk/server/static/assets/{chunk-4BX2VUAB-Be57QPgD.js → chunk-4BX2VUAB-B9drTgOW.js} +1 -1
- ksadk/server/static/assets/{chunk-55IACEB6-WjSuy4nv.js → chunk-55IACEB6-CK4ROATy.js} +1 -1
- ksadk/server/static/assets/{chunk-727SXJPM-BOovP-b5.js → chunk-727SXJPM-B9KtTTgw.js} +1 -1
- ksadk/server/static/assets/{chunk-AQP2D5EJ-DOmZNmWj.js → chunk-AQP2D5EJ-1O1mki3m.js} +1 -1
- ksadk/server/static/assets/{chunk-FMBD7UC4-DzwlodkU.js → chunk-FMBD7UC4-B_fH2YuH.js} +1 -1
- ksadk/server/static/assets/{chunk-ND2GUHAM-DMqAikEU.js → chunk-ND2GUHAM-D6rFLaVV.js} +1 -1
- ksadk/server/static/assets/chunk-QZHKN3VN-iXEZtKQc.js +1 -0
- ksadk/server/static/assets/classDiagram-4FO5ZUOK-DIveV17Q.js +1 -0
- ksadk/server/static/assets/classDiagram-v2-Q7XG4LA2-DIveV17Q.js +1 -0
- ksadk/server/static/assets/{cose-bilkent-S5V4N54A-Zhy2GBnP.js → cose-bilkent-S5V4N54A-ChNs_Mdk.js} +1 -1
- ksadk/server/static/assets/{dagre-BM42HDAG-CbKrtIUz.js → dagre-BM42HDAG-D0QXg_MQ.js} +1 -1
- ksadk/server/static/assets/{diagram-2AECGRRQ-BRApgQRK.js → diagram-2AECGRRQ-DBDDKwBQ.js} +1 -1
- ksadk/server/static/assets/{diagram-5GNKFQAL-D0xbnDiK.js → diagram-5GNKFQAL-BkwDOxmO.js} +1 -1
- ksadk/server/static/assets/{diagram-KO2AKTUF-CVugVI9L.js → diagram-KO2AKTUF-Bv8waYvY.js} +1 -1
- ksadk/server/static/assets/{diagram-LMA3HP47-DozUGgee.js → diagram-LMA3HP47-e1VECDdu.js} +1 -1
- ksadk/server/static/assets/{diagram-OG6HWLK6-ZOYsKlru.js → diagram-OG6HWLK6-Ds9adfRl.js} +1 -1
- ksadk/server/static/assets/{dist-DpKtt4Bo.js → dist-4N2NMVYg.js} +1 -1
- ksadk/server/static/assets/{dist-CkEKJFtk.js → dist-B-jNR6Vn.js} +1 -1
- ksadk/server/static/assets/{dist-D57wnGWd.js → dist-B5J2uhNk.js} +1 -1
- ksadk/server/static/assets/{dist-8_SfK1IP.js → dist-B5oS1zcj.js} +1 -1
- ksadk/server/static/assets/{dist-DMezvBGJ.js → dist-BB3-OwTQ.js} +1 -1
- ksadk/server/static/assets/{dist-2hInLS_J.js → dist-BIlKoW0k.js} +1 -1
- ksadk/server/static/assets/{dist-BrCcQ47T.js → dist-BjOdlDjj.js} +1 -1
- ksadk/server/static/assets/{dist-BmlMPsCc.js → dist-C1sz5Eo3.js} +1 -1
- ksadk/server/static/assets/{dist-B0NP_Z0z.js → dist-CWV4lDr1.js} +1 -1
- ksadk/server/static/assets/{dist-D-xOD048.js → dist-CmZHzlwZ.js} +1 -1
- ksadk/server/static/assets/{dist-DL_NKc_z.js → dist-D6MpCZtz.js} +1 -1
- ksadk/server/static/assets/{dist-DyOkZmze.js → dist-DJN_6Y39.js} +1 -1
- ksadk/server/static/assets/{dist-DmFaq_Iy.js → dist-DQU9EWw-.js} +1 -1
- ksadk/server/static/assets/{dist-gX6GOLKq.js → dist-DWKByxWV.js} +1 -1
- ksadk/server/static/assets/{dist-DyWsb78Q.js → dist-Dli7qW-B.js} +1 -1
- ksadk/server/static/assets/{dist-BIJVDMuB.js → dist-DsHPdk3W.js} +1 -1
- ksadk/server/static/assets/{dist-Dp0k7bWI.js → dist-X4TnqwOK.js} +1 -1
- ksadk/server/static/assets/{dist-8EEyGgDx.js → dist-_pHPTyZ-.js} +1 -1
- ksadk/server/static/assets/{dist-CPBtqRTW.js → dist-fhOuWds5.js} +1 -1
- ksadk/server/static/assets/{dist-D-JMdWPQ.js → dist-gLnrGsoU.js} +1 -1
- ksadk/server/static/assets/{dist-BssVmjIo.js → dist-j6Dxmf_j.js} +1 -1
- ksadk/server/static/assets/{dist-C3HIIaN5.js → dist-o2OVmKvt.js} +1 -1
- ksadk/server/static/assets/{erDiagram-TEJ5UH35-Bk-9bgE5.js → erDiagram-TEJ5UH35-BPItFKH6.js} +1 -1
- ksadk/server/static/assets/eventmodeling-FCH6USID-DYFSxbqr.js +1 -0
- ksadk/server/static/assets/{flowDiagram-I6XJVG4X-Dn9kZVuZ.js → flowDiagram-I6XJVG4X-BbYB9w5Y.js} +1 -1
- ksadk/server/static/assets/{ganttDiagram-6RSMTGT7-BGxXlJqm.js → ganttDiagram-6RSMTGT7-TQXa0KXv.js} +1 -1
- ksadk/server/static/assets/gitGraph-WXDBUCRP-DyNYp6BN.js +1 -0
- ksadk/server/static/assets/{gitGraphDiagram-PVQCEYII-C7Q0X-wi.js → gitGraphDiagram-PVQCEYII-bKblCnpa.js} +1 -1
- ksadk/server/static/assets/index-R52pJJvg.js +177 -0
- ksadk/server/static/assets/index-jJr3_955.css +2 -0
- ksadk/server/static/assets/info-J43DQDTF-WUx7qTde.js +1 -0
- ksadk/server/static/assets/{infoDiagram-5YYISTIA-BO_CSqq5.js → infoDiagram-5YYISTIA-BGVT7J_i.js} +1 -1
- ksadk/server/static/assets/{ishikawaDiagram-YF4QCWOH-eYLPLd46.js → ishikawaDiagram-YF4QCWOH-K0OOfVna.js} +1 -1
- ksadk/server/static/assets/{journeyDiagram-JHISSGLW-De9geSb0.js → journeyDiagram-JHISSGLW-8LeOJaZp.js} +1 -1
- ksadk/server/static/assets/{kanban-definition-UN3LZRKU-Cs80kQto.js → kanban-definition-UN3LZRKU-C6uCgZrW.js} +1 -1
- ksadk/server/static/assets/{katex-DSrCFD8n.js → katex-CYWseY5E.js} +1 -1
- ksadk/server/static/assets/{linear-CN9sbKTF.js → linear-hdBcYzM8.js} +1 -1
- ksadk/server/static/assets/{mermaid-parser.core-DlgSBU8l.js → mermaid-parser.core-BWrDjSZM.js} +2 -2
- ksadk/server/static/assets/{mindmap-definition-RKZ34NQL-9afz7Oug.js → mindmap-definition-RKZ34NQL-Bjy_2v4_.js} +1 -1
- ksadk/server/static/assets/packet-YPE3B663-BN13y_xl.js +1 -0
- ksadk/server/static/assets/pie-LRSECV5Y-ftSLXhhc.js +1 -0
- ksadk/server/static/assets/{pieDiagram-4H26LBE5-DPv_VstN.js → pieDiagram-4H26LBE5-GmmqNiVH.js} +1 -1
- ksadk/server/static/assets/{quadrantDiagram-W4KKPZXB-o77wZ1XK.js → quadrantDiagram-W4KKPZXB-BmkRbwCg.js} +1 -1
- ksadk/server/static/assets/radar-GUYGQ44K-jaICkIeO.js +1 -0
- ksadk/server/static/assets/{requirementDiagram-4Y6WPE33-BYVw81VC.js → requirementDiagram-4Y6WPE33-DSWFqL7c.js} +1 -1
- ksadk/server/static/assets/{sankeyDiagram-5OEKKPKP-B7nZFMKT.js → sankeyDiagram-5OEKKPKP-l6iXmU65.js} +1 -1
- ksadk/server/static/assets/{sequenceDiagram-3UESZ5HK-BGRRrFw5.js → sequenceDiagram-3UESZ5HK-Cq6UBeQB.js} +1 -1
- ksadk/server/static/assets/{stateDiagram-AJRCARHV-BUfHCoPs.js → stateDiagram-AJRCARHV-qzTQpg3Y.js} +1 -1
- ksadk/server/static/assets/stateDiagram-v2-BHNVJYJU-BYCoYGAN.js +1 -0
- ksadk/server/static/assets/{timeline-definition-PNZ67QCA-FbCTGrf9.js → timeline-definition-PNZ67QCA-DEz24vV0.js} +1 -1
- ksadk/server/static/assets/treeView-BLDUP644-Cpib_vlz.js +1 -0
- ksadk/server/static/assets/treemap-LRROVOQU-DQxfmVHK.js +1 -0
- ksadk/server/static/assets/{vennDiagram-CIIHVFJN-DWgwtPsJ.js → vennDiagram-CIIHVFJN-CMCixki2.js} +1 -1
- ksadk/server/static/assets/wardley-L42UT6IY-BQYHnnkU.js +1 -0
- ksadk/server/static/assets/{wardleyDiagram-YWT4CUSO-2x8G-tTM.js → wardleyDiagram-YWT4CUSO-CVzB8F9K.js} +1 -1
- ksadk/server/static/assets/{xychartDiagram-2RQKCTM6-Drma3AYl.js → xychartDiagram-2RQKCTM6-BFBqBLc1.js} +1 -1
- ksadk/server/static/index.html +2 -2
- ksadk/server/terminal_sessions.py +111 -58
- ksadk/sessions/__init__.py +69 -19
- ksadk/sessions/base.py +31 -11
- ksadk/sessions/continuity.py +2 -3
- ksadk/sessions/in_memory.py +46 -2
- ksadk/sessions/local_service.py +148 -17
- ksadk/sessions/postgres_service.py +143 -25
- ksadk/sessions/resilient.py +36 -8
- ksadk/skills/models.py +17 -5
- ksadk/skills/package_store.py +5 -2
- ksadk/skills/runtime/agent.py +20 -5
- ksadk/skills/runtime/artifacts.py +1 -5
- ksadk/skills/runtime/backends/e2b.py +10 -4
- ksadk/skills/runtime/backends/local.py +15 -3
- ksadk/skills/runtime/base.py +1 -2
- ksadk/skills/runtime/executor.py +10 -3
- ksadk/skills/runtime/factory.py +4 -2
- ksadk/skills/runtime/loader.py +1 -2
- ksadk/skills/runtime/registry.py +15 -10
- ksadk/skills/runtime/request.py +3 -1
- ksadk/skills/service_client.py +47 -16
- ksadk/skills/tool_defs.py +15 -10
- ksadk/studio/__init__.py +5 -0
- ksadk/studio/agent_avatar_assets.py +120 -0
- ksadk/studio/agent_lifecycle.py +51 -0
- ksadk/studio/api.py +1046 -0
- ksadk/studio/api_catalog_routes.py +324 -0
- ksadk/studio/api_contracts.py +177 -0
- ksadk/studio/api_helpers.py +132 -0
- ksadk/studio/authoring.py +524 -0
- ksadk/studio/authoring_coordinator.py +285 -0
- ksadk/studio/builder.py +320 -0
- ksadk/studio/capabilities.py +351 -0
- ksadk/studio/cloud.py +278 -0
- ksadk/studio/codex_agent_service.py +583 -0
- ksadk/studio/codex_builder.py +324 -0
- ksadk/studio/codex_manifest.py +278 -0
- ksadk/studio/codex_run.py +352 -0
- ksadk/studio/compiler.py +189 -0
- ksadk/studio/contracts.py +644 -0
- ksadk/studio/errors.py +47 -0
- ksadk/studio/evaluation.py +192 -0
- ksadk/studio/event_store.py +284 -0
- ksadk/studio/framework_run.py +114 -0
- ksadk/studio/identifiers.py +33 -0
- ksadk/studio/mcp_runtime.py +229 -0
- ksadk/studio/model_client.py +543 -0
- ksadk/studio/model_profile_service.py +214 -0
- ksadk/studio/operations.py +243 -0
- ksadk/studio/otel_trace.py +754 -0
- ksadk/studio/pagination.py +182 -0
- ksadk/studio/python_tool_inspection.py +183 -0
- ksadk/studio/repository.py +306 -0
- ksadk/studio/resource_catalog.py +1217 -0
- ksadk/studio/run_service.py +707 -0
- ksadk/studio/runtime_catalog.py +74 -0
- ksadk/studio/runtime_source.py +339 -0
- ksadk/studio/service.py +1141 -0
- ksadk/studio/shared_web.py +994 -0
- ksadk/studio/skill_discovery.py +628 -0
- ksadk/studio/static/assets/PrismRenderer-IYN-ffww.js +3 -0
- ksadk/studio/static/assets/index-CaEN0eTR.css +1 -0
- ksadk/studio/static/assets/index-DI2eY_NH.js +252 -0
- ksadk/studio/static/assets/react-vendor-BNmTiJ-I.js +9 -0
- ksadk/studio/static/assets/rolldown-runtime-hePW80VL.js +1 -0
- ksadk/studio/static/index.html +15 -0
- ksadk/studio/templates.py +630 -0
- ksadk/studio/validator.py +246 -0
- ksadk/studio/workspace.py +137 -0
- ksadk/terminal_exec_policy.py +3 -4
- ksadk/tools/gateway.py +101 -15
- ksadk/tools/result_budget.py +12 -4
- ksadk/toolsets/__init__.py +161 -37
- ksadk/toolsets/a2a.py +426 -0
- ksadk/toolsets/platform.py +21 -5
- ksadk/toolsets/sandbox.py +82 -16
- ksadk/toolsets/skills.py +200 -39
- ksadk/toolsets/web.py +123 -26
- ksadk/toolsets/workspace.py +212 -101
- ksadk/tracing/__init__.py +19 -21
- ksadk/tracing/exporters/inmemory_exporter.py +62 -51
- ksadk/tracing/setup.py +135 -506
- ksadk/tracing/span_utils.py +0 -41
- ksadk/tui/loop.py +78 -30
- ksadk/tui/stream_render.py +1 -0
- ksadk/ui_config.py +5 -3
- ksadk/version.py +1 -1
- ksadk_runtime_common/memory_backend/registry.py +56 -56
- ksadk_runtime_common/schemas/runtime_event_v1.json +95 -0
- ksadk_runtime_common/workspace_files/router.py +8 -2
- agentengine_sdk_python-0.7.0.dist-info/RECORD +0 -482
- ksadk/a2a/card_builder.py +0 -64
- ksadk/a2a/client.py +0 -309
- ksadk/runners/utils/langfuse.py +0 -230
- ksadk/server/static/assets/CodeBlock-P16VOLYi.js +0 -9
- ksadk/server/static/assets/MathMessageMarkdown-CdGiQWHR.js +0 -8
- ksadk/server/static/assets/MermaidBlock-IOcal8cm.js +0 -303
- ksadk/server/static/assets/NativeTerminalPanel-DGSDWsei.js +0 -1
- ksadk/server/static/assets/architecture-7EHR7CIX-BvUlsA6l.js +0 -1
- ksadk/server/static/assets/channel-CTuydARX.js +0 -1
- ksadk/server/static/assets/chunk-QZHKN3VN-BnCydivS.js +0 -1
- ksadk/server/static/assets/classDiagram-4FO5ZUOK-BNHg45Be.js +0 -1
- ksadk/server/static/assets/classDiagram-v2-Q7XG4LA2-BNHg45Be.js +0 -1
- ksadk/server/static/assets/eventmodeling-FCH6USID-DsAgWnlf.js +0 -1
- ksadk/server/static/assets/gitGraph-WXDBUCRP-BIao9Yjg.js +0 -1
- ksadk/server/static/assets/index-C00Lwg3L.js +0 -165
- ksadk/server/static/assets/index-DAURbCsP.css +0 -1
- ksadk/server/static/assets/info-J43DQDTF-CyipqGxU.js +0 -1
- ksadk/server/static/assets/packet-YPE3B663-CGkyRxlg.js +0 -1
- ksadk/server/static/assets/pie-LRSECV5Y-M0uWAf4n.js +0 -1
- ksadk/server/static/assets/radar-GUYGQ44K-Bdbm24-x.js +0 -1
- ksadk/server/static/assets/stateDiagram-v2-BHNVJYJU-8xBN_gLM.js +0 -1
- ksadk/server/static/assets/treeView-BLDUP644-npMo0ak1.js +0 -1
- ksadk/server/static/assets/treemap-LRROVOQU-CHdzrrdo.js +0 -1
- ksadk/server/static/assets/wardley-L42UT6IY-BEQz7wyy.js +0 -1
- ksadk/tracing/exporters/langfuse_exporter.py +0 -445
- {agentengine_sdk_python-0.7.0.dist-info → agentengine_sdk_python-0.8.1.dist-info}/entry_points.txt +0 -0
- {agentengine_sdk_python-0.7.0.dist-info → agentengine_sdk_python-0.8.1.dist-info}/licenses/LICENSE +0 -0
- {agentengine_sdk_python-0.7.0.dist-info → agentengine_sdk_python-0.8.1.dist-info}/top_level.txt +0 -0
|
@@ -0,0 +1,409 @@
|
|
|
1
|
+
"""Stable, serializable data objects shared by CLI and Studio evaluation."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from datetime import datetime, timezone
|
|
6
|
+
from enum import Enum
|
|
7
|
+
from typing import Any, Literal
|
|
8
|
+
|
|
9
|
+
from pydantic import BaseModel, ConfigDict, Field, model_validator
|
|
10
|
+
|
|
11
|
+
# ---------------------------------------------------------------------------
|
|
12
|
+
# Base model & helpers
|
|
13
|
+
# ---------------------------------------------------------------------------
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
def _to_camel_case(value: str) -> str:
|
|
17
|
+
head, *tail = value.split("_")
|
|
18
|
+
return head + "".join(part.capitalize() for part in tail)
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
def _content_digest(payload: dict[str, Any]) -> str:
|
|
22
|
+
import hashlib
|
|
23
|
+
import json
|
|
24
|
+
|
|
25
|
+
encoded = json.dumps(
|
|
26
|
+
payload,
|
|
27
|
+
ensure_ascii=False,
|
|
28
|
+
sort_keys=True,
|
|
29
|
+
separators=(",", ":"),
|
|
30
|
+
).encode("utf-8")
|
|
31
|
+
return hashlib.sha256(encoded).hexdigest()
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
class EvaluationModel(BaseModel):
|
|
35
|
+
"""Use camelCase on the wire while keeping Python fields snake_case."""
|
|
36
|
+
|
|
37
|
+
model_config = ConfigDict(
|
|
38
|
+
alias_generator=_to_camel_case,
|
|
39
|
+
populate_by_name=True,
|
|
40
|
+
extra="forbid",
|
|
41
|
+
)
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
# ---------------------------------------------------------------------------
|
|
45
|
+
# Enums
|
|
46
|
+
# ---------------------------------------------------------------------------
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
# Assertion kinds & data egress policy (configure how cases are asserted and
|
|
50
|
+
# what data may leave the local process).
|
|
51
|
+
class AssertionType(str, Enum):
|
|
52
|
+
"""Deterministic assertion kinds supported by the first evaluation phase."""
|
|
53
|
+
|
|
54
|
+
RESPONSE_EQUALS = "response.equals"
|
|
55
|
+
RESPONSE_CONTAINS = "response.contains"
|
|
56
|
+
RESPONSE_NOT_CONTAINS = "response.notContains"
|
|
57
|
+
RESPONSE_JSON_SCHEMA = "response.jsonSchema"
|
|
58
|
+
RUNTIME_MAX_LATENCY_MS = "runtime.maxLatencyMs"
|
|
59
|
+
RUNTIME_MAX_INPUT_TOKENS = "runtime.maxInputTokens"
|
|
60
|
+
RUNTIME_MAX_OUTPUT_TOKENS = "runtime.maxOutputTokens"
|
|
61
|
+
TOOL_CALLED = "tool.called"
|
|
62
|
+
TOOL_NOT_CALLED = "tool.notCalled"
|
|
63
|
+
|
|
64
|
+
|
|
65
|
+
class DataPolicy(str, Enum):
|
|
66
|
+
"""Controls what evaluation data may leave the local process."""
|
|
67
|
+
|
|
68
|
+
LOCAL_ONLY = "local_only"
|
|
69
|
+
METADATA_ONLY = "metadata_only"
|
|
70
|
+
REDACTED_TRACE = "redacted_trace"
|
|
71
|
+
FULL_TRACE = "full_trace"
|
|
72
|
+
|
|
73
|
+
|
|
74
|
+
# Lifecycle & outcome statuses (run-level, per-case target invocation, and
|
|
75
|
+
# per-metric verdicts).
|
|
76
|
+
class EvalRunStatus(str, Enum):
|
|
77
|
+
"""Lifecycle status for an entire evaluation run."""
|
|
78
|
+
|
|
79
|
+
PENDING = "PENDING"
|
|
80
|
+
RUNNING = "RUNNING"
|
|
81
|
+
PASSED = "PASSED"
|
|
82
|
+
FAILED = "FAILED"
|
|
83
|
+
ERROR = "ERROR"
|
|
84
|
+
CANCELLED = "CANCELLED"
|
|
85
|
+
|
|
86
|
+
|
|
87
|
+
class TargetRunStatus(str, Enum):
|
|
88
|
+
"""Terminal status returned by one target invocation."""
|
|
89
|
+
|
|
90
|
+
PASSED = "PASSED"
|
|
91
|
+
FAILED = "FAILED"
|
|
92
|
+
ERROR = "ERROR"
|
|
93
|
+
CANCELLED = "CANCELLED"
|
|
94
|
+
UNAVAILABLE = "UNAVAILABLE"
|
|
95
|
+
|
|
96
|
+
|
|
97
|
+
class MetricStatus(str, Enum):
|
|
98
|
+
"""Outcome of a single deterministic metric."""
|
|
99
|
+
|
|
100
|
+
PASS = "PASS"
|
|
101
|
+
FAIL = "FAIL"
|
|
102
|
+
UNAVAILABLE = "UNAVAILABLE"
|
|
103
|
+
ERROR = "ERROR"
|
|
104
|
+
|
|
105
|
+
|
|
106
|
+
# ---------------------------------------------------------------------------
|
|
107
|
+
# Evaluation set & input cases
|
|
108
|
+
# ---------------------------------------------------------------------------
|
|
109
|
+
|
|
110
|
+
|
|
111
|
+
class EvalTurn(EvaluationModel):
|
|
112
|
+
"""A single conversational turn within a multi-turn case."""
|
|
113
|
+
|
|
114
|
+
input: str = Field(min_length=1, max_length=32768)
|
|
115
|
+
expected_output: str | None = Field(default=None, max_length=32768)
|
|
116
|
+
expected_tools: list[dict[str, Any]] = Field(default_factory=list)
|
|
117
|
+
metadata: dict[str, Any] = Field(default_factory=dict)
|
|
118
|
+
|
|
119
|
+
|
|
120
|
+
class AssertionSpec(EvaluationModel):
|
|
121
|
+
"""A single deterministic assertion with type-dependent value validation."""
|
|
122
|
+
|
|
123
|
+
type: AssertionType
|
|
124
|
+
value: Any
|
|
125
|
+
required: bool = True
|
|
126
|
+
|
|
127
|
+
@model_validator(mode="after")
|
|
128
|
+
def validate_value(self) -> "AssertionSpec":
|
|
129
|
+
if self.type is AssertionType.RESPONSE_JSON_SCHEMA:
|
|
130
|
+
if not isinstance(self.value, dict):
|
|
131
|
+
raise ValueError("response.jsonSchema 的 value 必须是对象")
|
|
132
|
+
elif self.type.value.startswith("runtime."):
|
|
133
|
+
if isinstance(self.value, bool) or not isinstance(self.value, (int, float)):
|
|
134
|
+
raise ValueError(f"{self.type} 的 value 必须是非负数字")
|
|
135
|
+
if self.value < 0:
|
|
136
|
+
raise ValueError(f"{self.type} 的 value 必须是非负数字")
|
|
137
|
+
elif not isinstance(self.value, str):
|
|
138
|
+
raise ValueError(f"{self.type} 的 value 必须是字符串")
|
|
139
|
+
return self
|
|
140
|
+
|
|
141
|
+
|
|
142
|
+
class EvalCase(EvaluationModel):
|
|
143
|
+
"""One evaluation case: ordered turns plus assertions to check."""
|
|
144
|
+
|
|
145
|
+
id: str = Field(min_length=1, max_length=128)
|
|
146
|
+
turns: list[EvalTurn] = Field(min_length=1)
|
|
147
|
+
assertions: list[AssertionSpec] = Field(default_factory=list)
|
|
148
|
+
metadata: dict[str, Any] = Field(default_factory=dict)
|
|
149
|
+
|
|
150
|
+
@model_validator(mode="before")
|
|
151
|
+
@classmethod
|
|
152
|
+
def normalize_single_input(cls, value: Any) -> Any:
|
|
153
|
+
if not isinstance(value, dict):
|
|
154
|
+
return value
|
|
155
|
+
data = dict(value)
|
|
156
|
+
if "input" in data and "turns" not in data:
|
|
157
|
+
turn = {"input": data.pop("input")}
|
|
158
|
+
for field in ("expected_output", "expectedOutput", "expected_tools", "expectedTools"):
|
|
159
|
+
if field in data:
|
|
160
|
+
turn[field] = data.pop(field)
|
|
161
|
+
data["turns"] = [turn]
|
|
162
|
+
return data
|
|
163
|
+
|
|
164
|
+
@property
|
|
165
|
+
def input(self) -> str:
|
|
166
|
+
"""Return the final turn for legacy single-input consumers."""
|
|
167
|
+
|
|
168
|
+
return self.turns[-1].input
|
|
169
|
+
|
|
170
|
+
|
|
171
|
+
class EvalSetVersion(EvaluationModel):
|
|
172
|
+
"""A versioned, content-digested evaluation set of one or more cases."""
|
|
173
|
+
|
|
174
|
+
schema_version: Literal["ksadk.eval/v1"] = "ksadk.eval/v1"
|
|
175
|
+
name: str = Field(min_length=1, max_length=256)
|
|
176
|
+
cases: list[EvalCase] = Field(min_length=1)
|
|
177
|
+
metadata: dict[str, Any] = Field(default_factory=dict)
|
|
178
|
+
source_format: str = "native"
|
|
179
|
+
content_digest: str = ""
|
|
180
|
+
|
|
181
|
+
@model_validator(mode="after")
|
|
182
|
+
def validate_and_digest(self) -> "EvalSetVersion":
|
|
183
|
+
ids = [case.id for case in self.cases]
|
|
184
|
+
if len(ids) != len(set(ids)):
|
|
185
|
+
raise ValueError("评测集中的 case id 必须唯一")
|
|
186
|
+
expected = self.compute_digest()
|
|
187
|
+
if self.content_digest and self.content_digest != expected:
|
|
188
|
+
raise ValueError("contentDigest 与规范化评测集内容不一致")
|
|
189
|
+
self.content_digest = expected
|
|
190
|
+
return self
|
|
191
|
+
|
|
192
|
+
def compute_digest(self) -> str:
|
|
193
|
+
"""Hash normalized cases, excluding transport format and the digest itself."""
|
|
194
|
+
|
|
195
|
+
payload = self.model_dump(
|
|
196
|
+
mode="json",
|
|
197
|
+
by_alias=False,
|
|
198
|
+
exclude={"content_digest", "source_format"},
|
|
199
|
+
)
|
|
200
|
+
return _content_digest(payload)
|
|
201
|
+
|
|
202
|
+
|
|
203
|
+
# ---------------------------------------------------------------------------
|
|
204
|
+
# Target identity & references
|
|
205
|
+
# ---------------------------------------------------------------------------
|
|
206
|
+
|
|
207
|
+
|
|
208
|
+
class TargetKind(str, Enum):
|
|
209
|
+
"""Supported target categories; adapters own their implementation details."""
|
|
210
|
+
|
|
211
|
+
LOCAL_SOURCE = "local_source"
|
|
212
|
+
STUDIO_BUILD = "studio_build"
|
|
213
|
+
A2A = "a2a"
|
|
214
|
+
CODEX_WORKTREE = "codex_worktree"
|
|
215
|
+
|
|
216
|
+
|
|
217
|
+
class TargetSnapshot(EvaluationModel):
|
|
218
|
+
"""Immutable target identity recorded with an evaluation run."""
|
|
219
|
+
|
|
220
|
+
kind: TargetKind
|
|
221
|
+
entrypoint: str = Field(min_length=1, max_length=1024)
|
|
222
|
+
revision_digest: str = Field(min_length=1, max_length=128)
|
|
223
|
+
runtime: str = Field(default="", max_length=128)
|
|
224
|
+
metadata: dict[str, Any] = Field(default_factory=dict)
|
|
225
|
+
|
|
226
|
+
|
|
227
|
+
class TargetRef(EvaluationModel):
|
|
228
|
+
"""Unresolved target supplied by CLI/Studio before an adapter snapshots it."""
|
|
229
|
+
|
|
230
|
+
kind: TargetKind
|
|
231
|
+
locator: str = Field(min_length=1, max_length=2048)
|
|
232
|
+
entrypoint: str | None = Field(default=None, min_length=1, max_length=1024)
|
|
233
|
+
credential_ref: str | None = Field(default=None, min_length=1, max_length=512)
|
|
234
|
+
profile: str | None = Field(default=None, min_length=1, max_length=256)
|
|
235
|
+
|
|
236
|
+
|
|
237
|
+
# ---------------------------------------------------------------------------
|
|
238
|
+
# Execution config & request / spec
|
|
239
|
+
# ---------------------------------------------------------------------------
|
|
240
|
+
|
|
241
|
+
|
|
242
|
+
class EvaluationConfig(EvaluationModel):
|
|
243
|
+
"""Execution limits and evaluator selection shared by CLI and Studio."""
|
|
244
|
+
|
|
245
|
+
timeout_seconds: int = Field(default=120, ge=1, le=3600)
|
|
246
|
+
fail_fast: bool = False
|
|
247
|
+
evaluators: list[str] = Field(default_factory=list)
|
|
248
|
+
data_policy: DataPolicy = DataPolicy.LOCAL_ONLY
|
|
249
|
+
judge_model: str | None = Field(default=None, min_length=1, max_length=256)
|
|
250
|
+
judge_api_base: str | None = Field(default=None, min_length=1, max_length=2048)
|
|
251
|
+
judge_api_key_env: str = Field(
|
|
252
|
+
default="KSADK_EVAL_JUDGE_API_KEY",
|
|
253
|
+
pattern=r"^[A-Za-z_][A-Za-z0-9_]*$",
|
|
254
|
+
max_length=128,
|
|
255
|
+
)
|
|
256
|
+
|
|
257
|
+
|
|
258
|
+
class EvaluationRequest(EvaluationModel):
|
|
259
|
+
"""Validated input passed into the one public evaluation executor."""
|
|
260
|
+
|
|
261
|
+
evalset: EvalSetVersion
|
|
262
|
+
target: TargetRef
|
|
263
|
+
config: EvaluationConfig = Field(default_factory=EvaluationConfig)
|
|
264
|
+
report_dir: str | None = Field(default=None, min_length=1, max_length=2048)
|
|
265
|
+
|
|
266
|
+
|
|
267
|
+
class EvalRunSpec(EvaluationModel):
|
|
268
|
+
"""Immutable execution plan after the target has been snapshotted."""
|
|
269
|
+
|
|
270
|
+
id: str = Field(min_length=1, max_length=128)
|
|
271
|
+
evalset: EvalSetVersion
|
|
272
|
+
target: TargetSnapshot
|
|
273
|
+
config: EvaluationConfig = Field(default_factory=EvaluationConfig)
|
|
274
|
+
environment_digest: str = Field(default="", max_length=128)
|
|
275
|
+
attempt: int = Field(default=1, ge=1)
|
|
276
|
+
|
|
277
|
+
|
|
278
|
+
# ---------------------------------------------------------------------------
|
|
279
|
+
# Run results: trace, usage, target run, metric results, case run, report
|
|
280
|
+
# ---------------------------------------------------------------------------
|
|
281
|
+
|
|
282
|
+
|
|
283
|
+
# Per-case adapter output: trace linkage, token usage, and the normalized
|
|
284
|
+
# invocation result returned by a target adapter.
|
|
285
|
+
class TraceRef(EvaluationModel):
|
|
286
|
+
"""Queryable linkage to a recorded trace; at least one id required."""
|
|
287
|
+
|
|
288
|
+
run_id: str | None = None
|
|
289
|
+
trace_id: str | None = None
|
|
290
|
+
root_span_id: str | None = None
|
|
291
|
+
seq_id: int | None = Field(default=None, ge=1)
|
|
292
|
+
|
|
293
|
+
@model_validator(mode="after")
|
|
294
|
+
def require_reference(self) -> "TraceRef":
|
|
295
|
+
if not any((self.run_id, self.trace_id, self.seq_id)):
|
|
296
|
+
raise ValueError("TraceRef 至少需要一个可查询 ID")
|
|
297
|
+
return self
|
|
298
|
+
|
|
299
|
+
|
|
300
|
+
class UsageSnapshot(EvaluationModel):
|
|
301
|
+
"""Token usage for one invocation; zero unless the adapter reported it."""
|
|
302
|
+
|
|
303
|
+
input_tokens: int = Field(default=0, ge=0)
|
|
304
|
+
output_tokens: int = Field(default=0, ge=0)
|
|
305
|
+
total_tokens: int = Field(default=0, ge=0)
|
|
306
|
+
reported: bool = False
|
|
307
|
+
|
|
308
|
+
|
|
309
|
+
class TargetRun(EvaluationModel):
|
|
310
|
+
"""Normalized result returned by a target adapter for one case."""
|
|
311
|
+
|
|
312
|
+
status: TargetRunStatus
|
|
313
|
+
output: str = ""
|
|
314
|
+
duration_ms: int | None = Field(default=None, ge=0)
|
|
315
|
+
usage: UsageSnapshot = Field(default_factory=UsageSnapshot)
|
|
316
|
+
error_code: str | None = None
|
|
317
|
+
error_message: str | None = None
|
|
318
|
+
trace_ref: TraceRef | None = None
|
|
319
|
+
metadata: dict[str, Any] = Field(default_factory=dict)
|
|
320
|
+
|
|
321
|
+
|
|
322
|
+
# Evaluator output: per-metric verdicts, per-case aggregation, run-level
|
|
323
|
+
# summary counts, and the canonical persisted report.
|
|
324
|
+
class MetricResult(EvaluationModel):
|
|
325
|
+
"""One deterministic evaluator result and its non-sensitive evidence."""
|
|
326
|
+
|
|
327
|
+
name: str = Field(min_length=1, max_length=128)
|
|
328
|
+
version: str = Field(default="v1", min_length=1, max_length=32)
|
|
329
|
+
status: MetricStatus
|
|
330
|
+
score: float | None = Field(default=None, ge=0, le=1)
|
|
331
|
+
required: bool = True
|
|
332
|
+
evidence: dict[str, Any] = Field(default_factory=dict)
|
|
333
|
+
|
|
334
|
+
|
|
335
|
+
class CaseRun(EvaluationModel):
|
|
336
|
+
"""Target result and metric results for one case attempt."""
|
|
337
|
+
|
|
338
|
+
case_id: str = Field(min_length=1, max_length=128)
|
|
339
|
+
attempt: int = Field(default=1, ge=1)
|
|
340
|
+
target_run: TargetRun
|
|
341
|
+
metrics: list[MetricResult] = Field(default_factory=list)
|
|
342
|
+
|
|
343
|
+
@property
|
|
344
|
+
def passed(self) -> bool:
|
|
345
|
+
return self.target_run.status is TargetRunStatus.PASSED and all(
|
|
346
|
+
metric.status is MetricStatus.PASS for metric in self.metrics if metric.required
|
|
347
|
+
)
|
|
348
|
+
|
|
349
|
+
|
|
350
|
+
class EvalRunSummary(EvaluationModel):
|
|
351
|
+
"""Derived terminal counts; callers must not construct their own summary."""
|
|
352
|
+
|
|
353
|
+
total_cases: int = Field(default=0, ge=0)
|
|
354
|
+
passed_cases: int = Field(default=0, ge=0)
|
|
355
|
+
failed_cases: int = Field(default=0, ge=0)
|
|
356
|
+
unavailable_cases: int = Field(default=0, ge=0)
|
|
357
|
+
error_cases: int = Field(default=0, ge=0)
|
|
358
|
+
cancelled_cases: int = Field(default=0, ge=0)
|
|
359
|
+
|
|
360
|
+
|
|
361
|
+
class EvalRunReport(EvaluationModel):
|
|
362
|
+
"""Canonical persisted result consumed by CLI, Studio, and future sync."""
|
|
363
|
+
|
|
364
|
+
schema_version: Literal["ksadk.eval.report/v1"] = "ksadk.eval.report/v1"
|
|
365
|
+
spec: EvalRunSpec
|
|
366
|
+
status: EvalRunStatus
|
|
367
|
+
created_at: datetime = Field(default_factory=lambda: datetime.now(timezone.utc))
|
|
368
|
+
case_runs: list[CaseRun] = Field(default_factory=list)
|
|
369
|
+
summary: EvalRunSummary = Field(default_factory=EvalRunSummary)
|
|
370
|
+
report_digest: str = ""
|
|
371
|
+
|
|
372
|
+
@model_validator(mode="after")
|
|
373
|
+
def validate_case_ids(self) -> "EvalRunReport":
|
|
374
|
+
case_ids = [case.case_id for case in self.case_runs]
|
|
375
|
+
if len(case_ids) != len(set(case_ids)):
|
|
376
|
+
raise ValueError("一次评测中 case_id 不能重复")
|
|
377
|
+
self.summary = self._summarize_cases()
|
|
378
|
+
expected = self.compute_digest()
|
|
379
|
+
if self.report_digest and self.report_digest != expected:
|
|
380
|
+
raise ValueError("reportDigest 与规范化报告内容不一致")
|
|
381
|
+
self.report_digest = expected
|
|
382
|
+
return self
|
|
383
|
+
|
|
384
|
+
def _summarize_cases(self) -> EvalRunSummary:
|
|
385
|
+
counts = EvalRunSummary(total_cases=len(self.case_runs))
|
|
386
|
+
for case_run in self.case_runs:
|
|
387
|
+
if case_run.target_run.status is TargetRunStatus.ERROR or any(
|
|
388
|
+
metric.required and metric.status is MetricStatus.ERROR
|
|
389
|
+
for metric in case_run.metrics
|
|
390
|
+
):
|
|
391
|
+
counts.error_cases += 1
|
|
392
|
+
elif case_run.target_run.status is TargetRunStatus.CANCELLED:
|
|
393
|
+
counts.cancelled_cases += 1
|
|
394
|
+
elif case_run.target_run.status is TargetRunStatus.UNAVAILABLE or any(
|
|
395
|
+
metric.required and metric.status is MetricStatus.UNAVAILABLE
|
|
396
|
+
for metric in case_run.metrics
|
|
397
|
+
):
|
|
398
|
+
counts.unavailable_cases += 1
|
|
399
|
+
elif case_run.passed:
|
|
400
|
+
counts.passed_cases += 1
|
|
401
|
+
else:
|
|
402
|
+
counts.failed_cases += 1
|
|
403
|
+
return counts
|
|
404
|
+
|
|
405
|
+
def compute_digest(self) -> str:
|
|
406
|
+
"""Return a stable digest excluding the digest field itself."""
|
|
407
|
+
|
|
408
|
+
payload = self.model_dump(mode="json", by_alias=False, exclude={"report_digest"})
|
|
409
|
+
return _content_digest(payload)
|