google-agents-cli-extensions 1.14.2__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- core/claude/README.md +35 -0
- core/claude/template/.agents/skills/agents-cli-claude/SKILL.md +106 -0
- core/claude/template/.agents/skills/agents-cli-claude/references/claude.md +386 -0
- core/claude/template/.agents/skills/agents-cli-claude/references/migrate-claude-agent.md +318 -0
- core/claude/template/.agents/skills/agents-cli-claude/references/migrate-from-aws.md +132 -0
- core/claude/template/.dockerignore +28 -0
- core/claude/template/.env +18 -0
- core/claude/template/.env.example +17 -0
- core/claude/template/.template/templateconfig.yaml +31 -0
- core/claude/template/AGENTS.md +33 -0
- core/claude/template/Dockerfile +55 -0
- core/claude/template/agent_config/README.md +9 -0
- core/claude/template/agents-cli-extension.yaml +49 -0
- core/claude/template/app/__init__.py +15 -0
- core/claude/template/app/agent.py +49 -0
- core/claude/template/app/app_utils/a2a.py +245 -0
- core/claude/template/app/app_utils/bq_analytics/__init__.py +124 -0
- core/claude/template/app/app_utils/bq_analytics/recorder.py +593 -0
- core/claude/template/app/app_utils/bq_analytics/redaction.py +209 -0
- core/claude/template/app/app_utils/bq_analytics/schema.py +394 -0
- core/claude/template/app/app_utils/bq_analytics/tables.py +104 -0
- core/claude/template/app/app_utils/bq_analytics/uploader.py +188 -0
- core/claude/template/app/app_utils/claude.py +297 -0
- core/claude/template/app/app_utils/completions.py +225 -0
- core/claude/template/app/app_utils/conversation.py +29 -0
- core/claude/template/app/app_utils/memory_bank.py +308 -0
- core/claude/template/app/app_utils/project_settings.py +91 -0
- core/claude/template/app/app_utils/reasoning_engine.py +237 -0
- core/claude/template/app/app_utils/remote_agents.py +349 -0
- core/claude/template/app/app_utils/remote_auth.py +159 -0
- core/claude/template/app/app_utils/replies.py +143 -0
- core/claude/template/app/app_utils/session_store.py +174 -0
- core/claude/template/app/app_utils/settings.py +24 -0
- core/claude/template/app/app_utils/telemetry/__init__.py +114 -0
- core/claude/template/app/app_utils/telemetry/call_counts.py +44 -0
- core/claude/template/app/app_utils/telemetry/cli_relay.py +266 -0
- core/claude/template/app/app_utils/telemetry/config.py +133 -0
- core/claude/template/app/app_utils/telemetry/logs.py +175 -0
- core/claude/template/app/app_utils/telemetry/metrics.py +270 -0
- core/claude/template/app/app_utils/telemetry/request_flush.py +104 -0
- core/claude/template/app/app_utils/telemetry/subagent_spans.py +80 -0
- core/claude/template/app/app_utils/telemetry/traces.py +166 -0
- core/claude/template/app/app_utils/turn_events.py +463 -0
- core/claude/template/app/app_utils/turn_options.py +219 -0
- core/claude/template/app/app_utils/workspaces.py +65 -0
- core/claude/template/app/fast_api_app.py +71 -0
- core/claude/template/app/feature_tools.py +27 -0
- core/claude/template/pyproject.toml +100 -0
- core/claude/template/scripts/eval_generate.py +340 -0
- core/claude/template/scripts/publish.py +66 -0
- core/claude/template/scripts/run.py +85 -0
- core/claude/template/scripts/unsupported.py +48 -0
- core/claude/template/tests/eval/datasets/basic-dataset.json +31 -0
- core/claude/template/tests/eval/eval_config.yaml +17 -0
- core/claude/template/tests/eval/response_quality.py +60 -0
- core/claude/template/tests/integration/test_server_e2e.py +168 -0
- core/claude/template/tests/load_test/.results/.placeholder +0 -0
- core/claude/template/tests/load_test/README.md +24 -0
- core/claude/template/tests/load_test/load_test.py +118 -0
- core/claude/template/tests/unit/conftest.py +63 -0
- core/claude/template/tests/unit/helpers.py +32 -0
- core/claude/template/tests/unit/test_a2a.py +432 -0
- core/claude/template/tests/unit/test_bq_analytics.py +928 -0
- core/claude/template/tests/unit/test_bq_redaction.py +89 -0
- core/claude/template/tests/unit/test_claude.py +825 -0
- core/claude/template/tests/unit/test_completions.py +369 -0
- core/claude/template/tests/unit/test_fast_api_app.py +59 -0
- core/claude/template/tests/unit/test_feature_tools.py +55 -0
- core/claude/template/tests/unit/test_memory_bank.py +625 -0
- core/claude/template/tests/unit/test_reasoning_engine.py +295 -0
- core/claude/template/tests/unit/test_remote_agents.py +517 -0
- core/claude/template/tests/unit/test_telemetry.py +991 -0
- core/claude/template/tests/unit/test_turn_events.py +309 -0
- core/claude/template/tests/unit/test_turn_options.py +394 -0
- core/claude/template/uv.lock +3029 -0
- core/langchain/README.md +54 -0
- core/langchain/template/.agents/skills/agents-cli-langchain/SKILL.md +84 -0
- core/langchain/template/.agents/skills/agents-cli-langchain/references/langchain.md +120 -0
- core/langchain/template/.agents/skills/agents-cli-langchain/references/samples.md +42 -0
- core/langchain/template/.template/templateconfig.yaml +32 -0
- core/langchain/template/AGENTS.md +35 -0
- core/langchain/template/agents-cli-extension.yaml +54 -0
- core/langchain/template/app/__init__.py +19 -0
- core/langchain/template/app/agent.py +61 -0
- core/langchain/template/app/app_utils/a2a.py +176 -0
- core/langchain/template/app/app_utils/content.py +37 -0
- core/langchain/template/app/app_utils/telemetry.py +154 -0
- core/langchain/template/app/fast_api_app.py +68 -0
- core/langchain/template/langgraph.json +7 -0
- core/langchain/template/pyproject.toml +102 -0
- core/langchain/template/scripts/_content.py +40 -0
- core/langchain/template/scripts/eval_generate.py +202 -0
- core/langchain/template/scripts/publish.py +59 -0
- core/langchain/template/scripts/run.py +59 -0
- core/langchain/template/scripts/unsupported.py +48 -0
- core/langchain/template/tests/eval/datasets/basic-dataset.json +16 -0
- core/langchain/template/tests/eval/eval_config.yaml +32 -0
- core/langchain/template/tests/eval/response_quality.py +60 -0
- core/langchain/template/tests/integration/test_server_e2e.py +173 -0
- core/langchain/template/tests/load_test/.results/.placeholder +0 -0
- core/langchain/template/tests/load_test/README.md +20 -0
- core/langchain/template/tests/load_test/load_test.py +82 -0
- core/langchain/template/uv.lock +3820 -0
- google_agents_cli_extensions-1.14.2.dist-info/METADATA +133 -0
- google_agents_cli_extensions-1.14.2.dist-info/RECORD +107 -0
- google_agents_cli_extensions-1.14.2.dist-info/WHEEL +4 -0
- google_agents_cli_extensions-1.14.2.dist-info/licenses/LICENSE +201 -0
|
@@ -0,0 +1,65 @@
|
|
|
1
|
+
# Copyright 2026 Google LLC
|
|
2
|
+
#
|
|
3
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
4
|
+
# you may not use this file except in compliance with the License.
|
|
5
|
+
# You may obtain a copy of the License at
|
|
6
|
+
#
|
|
7
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
|
8
|
+
#
|
|
9
|
+
# Unless required by applicable law or agreed to in writing, software
|
|
10
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
11
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
12
|
+
# See the License for the specific language governing permissions and
|
|
13
|
+
# limitations under the License.
|
|
14
|
+
|
|
15
|
+
"""Per-conversation working directories, and evicting the least recently used."""
|
|
16
|
+
|
|
17
|
+
from __future__ import annotations
|
|
18
|
+
|
|
19
|
+
import os
|
|
20
|
+
import shutil
|
|
21
|
+
import tempfile
|
|
22
|
+
from pathlib import Path
|
|
23
|
+
|
|
24
|
+
import claude_agent_sdk
|
|
25
|
+
|
|
26
|
+
# One working directory per agent and conversation. The CLI keys session
|
|
27
|
+
# transcripts by working directory, so a conversation must always land in the same one.
|
|
28
|
+
WORKSPACES = Path(tempfile.gettempdir()) / "claude-agent"
|
|
29
|
+
|
|
30
|
+
# Workspaces kept per instance before the least recently used are deleted. On
|
|
31
|
+
# Cloud Run the temp dir is memory. With Cloud SQL, evicted conversations resume.
|
|
32
|
+
MAX_WORKSPACES = int(os.environ.get("CLAUDE_MAX_WORKSPACES", "100"))
|
|
33
|
+
|
|
34
|
+
# This process's workspaces, least recently used first, with their running turn
|
|
35
|
+
# counts. Only these are evicted, since other processes on the host share WORKSPACES.
|
|
36
|
+
_workspaces: dict[Path, int] = {}
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
def claim_workspace(workspace: Path) -> None:
|
|
40
|
+
"""Mark a turn running in `workspace`, evicting idle ones beyond MAX_WORKSPACES."""
|
|
41
|
+
_workspaces[workspace] = _workspaces.pop(workspace, 0) + 1
|
|
42
|
+
idle = [w for w, turns in _workspaces.items() if not turns]
|
|
43
|
+
for evicted in idle[: max(0, len(_workspaces) - MAX_WORKSPACES)]:
|
|
44
|
+
del _workspaces[evicted]
|
|
45
|
+
remove_workspace(evicted)
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
def release_workspace(workspace: Path) -> None:
|
|
49
|
+
"""Mark a turn in `workspace` finished, so it can be evicted once idle."""
|
|
50
|
+
_workspaces[workspace] -= 1
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
def remove_workspace(workspace: Path) -> None:
|
|
54
|
+
"""Delete a workspace and the transcripts the CLI kept for it."""
|
|
55
|
+
transcripts = transcripts_dir(workspace)
|
|
56
|
+
shutil.rmtree(workspace, ignore_errors=True)
|
|
57
|
+
shutil.rmtree(transcripts, ignore_errors=True)
|
|
58
|
+
|
|
59
|
+
|
|
60
|
+
def transcripts_dir(workspace: Path) -> Path:
|
|
61
|
+
"""Where the CLI keeps local transcripts for sessions run in `workspace`."""
|
|
62
|
+
config_home = Path(os.environ.get("CLAUDE_CONFIG_DIR") or Path.home() / ".claude")
|
|
63
|
+
return (
|
|
64
|
+
config_home / "projects" / claude_agent_sdk.project_key_for_directory(workspace)
|
|
65
|
+
)
|
|
@@ -0,0 +1,71 @@
|
|
|
1
|
+
# Copyright 2026 Google LLC
|
|
2
|
+
#
|
|
3
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
4
|
+
# you may not use this file except in compliance with the License.
|
|
5
|
+
# You may obtain a copy of the License at
|
|
6
|
+
#
|
|
7
|
+
# https://www.apache.org/licenses/LICENSE-2.0
|
|
8
|
+
#
|
|
9
|
+
# Unless required by applicable law or agreed to in writing, software
|
|
10
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
11
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
12
|
+
# See the License for the specific language governing permissions and
|
|
13
|
+
# limitations under the License.
|
|
14
|
+
|
|
15
|
+
"""FastAPI app serving the agent over A2A at ``/a2a/{name}``, and to Agent Runtime's
|
|
16
|
+
``:query`` and ``:streamQuery`` methods."""
|
|
17
|
+
|
|
18
|
+
from __future__ import annotations
|
|
19
|
+
|
|
20
|
+
import os
|
|
21
|
+
from collections.abc import AsyncIterator
|
|
22
|
+
from contextlib import asynccontextmanager
|
|
23
|
+
|
|
24
|
+
from fastapi import FastAPI
|
|
25
|
+
|
|
26
|
+
from app.app_utils import bq_analytics
|
|
27
|
+
from app.app_utils.telemetry import (
|
|
28
|
+
FASTAPI_TELEMETRY,
|
|
29
|
+
FlushTelemetryMiddleware,
|
|
30
|
+
setup_telemetry,
|
|
31
|
+
)
|
|
32
|
+
|
|
33
|
+
setup_telemetry()
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
@asynccontextmanager
|
|
37
|
+
async def lifespan(app_instance: FastAPI) -> AsyncIterator[None]:
|
|
38
|
+
# Imported here so app.agent sees the .env that setup_telemetry() loaded.
|
|
39
|
+
from app.app_utils.a2a import add_a2a_routes
|
|
40
|
+
from app.app_utils.reasoning_engine import add_reasoning_engine_routes
|
|
41
|
+
|
|
42
|
+
add_a2a_routes(app_instance)
|
|
43
|
+
add_reasoning_engine_routes(app_instance)
|
|
44
|
+
# Create the BigQuery Agent Analytics table up front, and flush rows on shutdown.
|
|
45
|
+
bq_analytics.writer()
|
|
46
|
+
yield
|
|
47
|
+
bq_analytics.flush(5)
|
|
48
|
+
|
|
49
|
+
|
|
50
|
+
app = FastAPI(
|
|
51
|
+
title="claude-agent",
|
|
52
|
+
description="Claude Agent SDK agent (A2A protocol)",
|
|
53
|
+
lifespan=lifespan,
|
|
54
|
+
telemetry=FASTAPI_TELEMETRY,
|
|
55
|
+
)
|
|
56
|
+
app.add_middleware(FlushTelemetryMiddleware)
|
|
57
|
+
|
|
58
|
+
|
|
59
|
+
@app.get("/health")
|
|
60
|
+
def health() -> dict[str, str]:
|
|
61
|
+
return {"status": "ok"}
|
|
62
|
+
|
|
63
|
+
|
|
64
|
+
if __name__ == "__main__":
|
|
65
|
+
import uvicorn
|
|
66
|
+
|
|
67
|
+
uvicorn.run(
|
|
68
|
+
app,
|
|
69
|
+
host=os.environ.get("HOST", "127.0.0.1"),
|
|
70
|
+
port=int(os.environ.get("PORT", "8080")),
|
|
71
|
+
)
|
|
@@ -0,0 +1,27 @@
|
|
|
1
|
+
# Copyright 2026 Google LLC
|
|
2
|
+
#
|
|
3
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
4
|
+
# you may not use this file except in compliance with the License.
|
|
5
|
+
# You may obtain a copy of the License at
|
|
6
|
+
#
|
|
7
|
+
# https://www.apache.org/licenses/LICENSE-2.0
|
|
8
|
+
#
|
|
9
|
+
# Unless required by applicable law or agreed to in writing, software
|
|
10
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
11
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
12
|
+
# See the License for the specific language governing permissions and
|
|
13
|
+
# limitations under the License.
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
"""Tools that scaffold features add to root_agent, each only while its feature is on."""
|
|
17
|
+
|
|
18
|
+
from claude_agent_sdk import McpServerConfig
|
|
19
|
+
|
|
20
|
+
from app.app_utils import memory_bank
|
|
21
|
+
|
|
22
|
+
FEATURE_MCP_SERVERS: dict[str, McpServerConfig] = {
|
|
23
|
+
**(memory_bank.mcp_servers() if memory_bank.enabled() else {}),
|
|
24
|
+
}
|
|
25
|
+
FEATURE_TOOLS: list[str] = [
|
|
26
|
+
*(memory_bank.allowed_tools() if memory_bank.enabled() else []),
|
|
27
|
+
]
|
|
@@ -0,0 +1,100 @@
|
|
|
1
|
+
[project]
|
|
2
|
+
name = "claude-agent"
|
|
3
|
+
version = "0.1.0"
|
|
4
|
+
description = ""
|
|
5
|
+
authors = [
|
|
6
|
+
{name = "Your Name", email = "your@email.com"},
|
|
7
|
+
]
|
|
8
|
+
dependencies = [
|
|
9
|
+
"claude-agent-sdk>=0.2.158,<0.3.0",
|
|
10
|
+
"anyio>=4.0.0,<5.0.0",
|
|
11
|
+
"google-auth>=2.61.0,<3.0.0",
|
|
12
|
+
"asyncpg>=0.30.0,<1.0.0",
|
|
13
|
+
"python-dotenv>=1.0.0,<2.0.0",
|
|
14
|
+
"fastapi>=0.142.0,<1.0.0",
|
|
15
|
+
"uvicorn~=0.34.0",
|
|
16
|
+
"a2a-sdk[http-server]>=1.0,<2",
|
|
17
|
+
"google-cloud-bigquery>=3.0.0,<4.0.0",
|
|
18
|
+
"google-cloud-aiplatform>=2.2.0,<3.0.0",
|
|
19
|
+
"opentelemetry-exporter-otlp-proto-http>=1.30.0,<2.0.0",
|
|
20
|
+
"opentelemetry-resourcedetector-gcp>=1.6.0a0,<2",
|
|
21
|
+
"openinference-instrumentation-claude-agent-sdk>=0.1.21,<0.2",
|
|
22
|
+
"opentelemetry-util-genai[upload]>=1.2b0,<2",
|
|
23
|
+
"gcsfs>=2025.9.0",
|
|
24
|
+
]
|
|
25
|
+
requires-python = ">=3.11,<3.14"
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
[dependency-groups]
|
|
29
|
+
dev = [
|
|
30
|
+
"pytest>=9.0.2,<10.0.0",
|
|
31
|
+
"pytest-asyncio>=1.0.0,<2.0.0",
|
|
32
|
+
"nest-asyncio>=1.6.0,<2.0.0",
|
|
33
|
+
]
|
|
34
|
+
|
|
35
|
+
[project.optional-dependencies]
|
|
36
|
+
eval = [
|
|
37
|
+
"google-cloud-aiplatform[evaluation]>=2.2.0,<3.0.0",
|
|
38
|
+
]
|
|
39
|
+
lint = [
|
|
40
|
+
"ruff==0.15.13",
|
|
41
|
+
"ty>=0.0.1a0",
|
|
42
|
+
"codespell>=2.2.0,<3.0.0",
|
|
43
|
+
]
|
|
44
|
+
|
|
45
|
+
[tool.ruff]
|
|
46
|
+
line-length = 88
|
|
47
|
+
target-version = "py311"
|
|
48
|
+
|
|
49
|
+
[tool.ruff.lint]
|
|
50
|
+
select = [
|
|
51
|
+
"E", # pycodestyle
|
|
52
|
+
"F", # pyflakes
|
|
53
|
+
"W", # pycodestyle warnings
|
|
54
|
+
"I", # isort
|
|
55
|
+
"C", # flake8-comprehensions
|
|
56
|
+
"B", # flake8-bugbear
|
|
57
|
+
"UP", # pyupgrade
|
|
58
|
+
"RUF", # ruff specific rules
|
|
59
|
+
]
|
|
60
|
+
ignore = ["E501", "C901", "B006"] # ignore line too long, too complex
|
|
61
|
+
|
|
62
|
+
[tool.ruff.lint.isort]
|
|
63
|
+
known-first-party = ["app", "frontend"]
|
|
64
|
+
|
|
65
|
+
[tool.ty]
|
|
66
|
+
# ty is Astral's Rust-based type checker (same team as ruff/uv)
|
|
67
|
+
# See: https://docs.astral.sh/ty/
|
|
68
|
+
|
|
69
|
+
[tool.ty.environment]
|
|
70
|
+
python-version = "3.10"
|
|
71
|
+
|
|
72
|
+
[tool.ty.src]
|
|
73
|
+
exclude = [".venv/**"]
|
|
74
|
+
|
|
75
|
+
[tool.ty.rules]
|
|
76
|
+
unresolved-import = "ignore"
|
|
77
|
+
unresolved-attribute = "ignore"
|
|
78
|
+
invalid-argument-type = "ignore"
|
|
79
|
+
invalid-assignment = "ignore"
|
|
80
|
+
invalid-return-type = "ignore"
|
|
81
|
+
possibly-missing-attribute = "ignore"
|
|
82
|
+
not-subscriptable = "ignore"
|
|
83
|
+
deprecated = "ignore"
|
|
84
|
+
|
|
85
|
+
[tool.codespell]
|
|
86
|
+
ignore-words-list = "rouge"
|
|
87
|
+
skip = "./locust_env/*,uv.lock,.venv,./frontend,**/package-lock.json"
|
|
88
|
+
|
|
89
|
+
|
|
90
|
+
[build-system]
|
|
91
|
+
requires = ["hatchling"]
|
|
92
|
+
build-backend = "hatchling.build"
|
|
93
|
+
|
|
94
|
+
|
|
95
|
+
[tool.pytest.ini_options]
|
|
96
|
+
pythonpath = "."
|
|
97
|
+
asyncio_default_fixture_loop_scope = "function"
|
|
98
|
+
|
|
99
|
+
[tool.hatch.build.targets.wheel]
|
|
100
|
+
packages = ["app"]
|
|
@@ -0,0 +1,340 @@
|
|
|
1
|
+
# Copyright 2026 Google LLC
|
|
2
|
+
#
|
|
3
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
4
|
+
# you may not use this file except in compliance with the License.
|
|
5
|
+
# You may obtain a copy of the License at
|
|
6
|
+
#
|
|
7
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
|
8
|
+
#
|
|
9
|
+
# Unless required by applicable law or agreed to in writing, software
|
|
10
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
11
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
12
|
+
# See the License for the specific language governing permissions and
|
|
13
|
+
# limitations under the License.
|
|
14
|
+
|
|
15
|
+
"""Generate eval traces for `agents-cli eval generate`.
|
|
16
|
+
|
|
17
|
+
Output is the EvaluationDataset shape `agents-cli eval grade` reads, with tool
|
|
18
|
+
calls as `function_call` / `function_response` parts.
|
|
19
|
+
"""
|
|
20
|
+
|
|
21
|
+
from __future__ import annotations
|
|
22
|
+
|
|
23
|
+
import argparse
|
|
24
|
+
import datetime
|
|
25
|
+
import json
|
|
26
|
+
import logging
|
|
27
|
+
import os
|
|
28
|
+
import sys
|
|
29
|
+
import uuid
|
|
30
|
+
from collections.abc import Callable
|
|
31
|
+
from pathlib import Path
|
|
32
|
+
from typing import Any
|
|
33
|
+
|
|
34
|
+
# `app` may not be installed as a package, so import it from the project root.
|
|
35
|
+
sys.path.insert(0, str(Path(__file__).resolve().parent.parent))
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
def _resolve_output_path(output: str) -> Path:
|
|
39
|
+
p = Path(output)
|
|
40
|
+
# A trailing separator means a directory that may not exist yet.
|
|
41
|
+
if p.is_dir() or output.endswith(("/", os.sep)):
|
|
42
|
+
p.mkdir(parents=True, exist_ok=True)
|
|
43
|
+
stamp = datetime.datetime.now().strftime("%Y%m%d_%H%M%S")
|
|
44
|
+
return p / f"eval_dataset_{stamp}.json"
|
|
45
|
+
p.parent.mkdir(parents=True, exist_ok=True)
|
|
46
|
+
return p
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
def _user_prompt(turn: dict) -> str | None:
|
|
50
|
+
for event in reversed(turn.get("events") or []):
|
|
51
|
+
if event.get("author") == "user":
|
|
52
|
+
for part in event.get("content", {}).get("parts") or []:
|
|
53
|
+
if "text" in part:
|
|
54
|
+
return part["text"]
|
|
55
|
+
return None
|
|
56
|
+
|
|
57
|
+
|
|
58
|
+
def _prompt_text(prompt: object) -> str | None:
|
|
59
|
+
"""Text of a case's top-level ``prompt`` (a Content: role + parts)."""
|
|
60
|
+
if isinstance(prompt, str):
|
|
61
|
+
return prompt
|
|
62
|
+
if isinstance(prompt, dict):
|
|
63
|
+
for part in prompt.get("parts") or []:
|
|
64
|
+
if isinstance(part, dict) and part.get("text"):
|
|
65
|
+
return part["text"]
|
|
66
|
+
return None
|
|
67
|
+
|
|
68
|
+
|
|
69
|
+
def _agent_events(
|
|
70
|
+
agent_name: str, reply: str, tool_calls: list[dict[str, Any]]
|
|
71
|
+
) -> list[dict]:
|
|
72
|
+
"""The agent's side of a turn: each tool call and result, then the reply.
|
|
73
|
+
|
|
74
|
+
A subagent's calls follow the call that delegated to them.
|
|
75
|
+
"""
|
|
76
|
+
events = _call_events(agent_name, None, tool_calls)
|
|
77
|
+
events.append(
|
|
78
|
+
{"author": agent_name, "content": {"role": "model", "parts": [{"text": reply}]}}
|
|
79
|
+
)
|
|
80
|
+
return events
|
|
81
|
+
|
|
82
|
+
|
|
83
|
+
def _call_events(
|
|
84
|
+
agent_name: str, parent: str | None, tool_calls: list[dict[str, Any]]
|
|
85
|
+
) -> list[dict]:
|
|
86
|
+
events: list[dict] = []
|
|
87
|
+
for call in tool_calls:
|
|
88
|
+
if call.get("parent") != parent:
|
|
89
|
+
continue
|
|
90
|
+
author = call.get("agent", agent_name)
|
|
91
|
+
events.append(
|
|
92
|
+
{
|
|
93
|
+
"author": author,
|
|
94
|
+
"content": {
|
|
95
|
+
"role": "model",
|
|
96
|
+
"parts": [
|
|
97
|
+
{
|
|
98
|
+
"function_call": {
|
|
99
|
+
"id": call["id"],
|
|
100
|
+
"name": call["name"],
|
|
101
|
+
"args": call["args"],
|
|
102
|
+
}
|
|
103
|
+
}
|
|
104
|
+
],
|
|
105
|
+
},
|
|
106
|
+
}
|
|
107
|
+
)
|
|
108
|
+
events.append(
|
|
109
|
+
{
|
|
110
|
+
"author": author,
|
|
111
|
+
"content": {
|
|
112
|
+
"role": "user",
|
|
113
|
+
"parts": [
|
|
114
|
+
{
|
|
115
|
+
"function_response": {
|
|
116
|
+
"id": call["id"],
|
|
117
|
+
"name": call["name"],
|
|
118
|
+
"response": {"result": call.get("result", "")},
|
|
119
|
+
}
|
|
120
|
+
}
|
|
121
|
+
],
|
|
122
|
+
},
|
|
123
|
+
}
|
|
124
|
+
)
|
|
125
|
+
events.extend(_call_events(agent_name, call["id"], tool_calls))
|
|
126
|
+
return events
|
|
127
|
+
|
|
128
|
+
|
|
129
|
+
async def _tool_declarations(options: Any) -> list[dict]:
|
|
130
|
+
"""Function declarations for the agent's tools.
|
|
131
|
+
|
|
132
|
+
External MCP servers are skipped; built-in tools get a name-only declaration.
|
|
133
|
+
"""
|
|
134
|
+
import anyio
|
|
135
|
+
from mcp import ClientSession
|
|
136
|
+
from mcp.shared.memory import create_client_server_memory_streams
|
|
137
|
+
|
|
138
|
+
declarations: list[dict] = []
|
|
139
|
+
for server_name, config in (options.mcp_servers or {}).items():
|
|
140
|
+
if not isinstance(config, dict) or config.get("type") != "sdk":
|
|
141
|
+
continue
|
|
142
|
+
server = config["instance"]
|
|
143
|
+
async with create_client_server_memory_streams() as (client, served):
|
|
144
|
+
async with anyio.create_task_group() as tg:
|
|
145
|
+
tg.start_soon(
|
|
146
|
+
server.run, *served, server.create_initialization_options()
|
|
147
|
+
)
|
|
148
|
+
async with ClientSession(*client) as session:
|
|
149
|
+
await session.initialize()
|
|
150
|
+
listed = await session.list_tools()
|
|
151
|
+
tg.cancel_scope.cancel()
|
|
152
|
+
declarations.extend(
|
|
153
|
+
{
|
|
154
|
+
"name": f"mcp__{server_name}__{tool.name}",
|
|
155
|
+
"description": tool.description or "",
|
|
156
|
+
"parameters": tool.model_dump(by_alias=True)["inputSchema"],
|
|
157
|
+
}
|
|
158
|
+
for tool in listed.tools
|
|
159
|
+
)
|
|
160
|
+
if isinstance(options.tools, list):
|
|
161
|
+
declarations.extend({"name": name} for name in options.tools)
|
|
162
|
+
return declarations
|
|
163
|
+
|
|
164
|
+
|
|
165
|
+
def agents_metadata(options: Any, agent_name: str) -> dict:
|
|
166
|
+
"""`agent_data.agents` for the traces: instructions and tools per agent.
|
|
167
|
+
|
|
168
|
+
Tool metrics grade calls against these; on failure this warns and returns {}.
|
|
169
|
+
"""
|
|
170
|
+
import anyio
|
|
171
|
+
|
|
172
|
+
try:
|
|
173
|
+
declarations = anyio.run(_tool_declarations, options)
|
|
174
|
+
except Exception as e:
|
|
175
|
+
logging.warning(
|
|
176
|
+
"Could not list the agent's tools (%s: %s); traces will omit "
|
|
177
|
+
"agent_data.agents -- tool metrics will degrade.",
|
|
178
|
+
type(e).__name__,
|
|
179
|
+
e,
|
|
180
|
+
)
|
|
181
|
+
return {}
|
|
182
|
+
config: dict[str, Any] = {"agent_id": agent_name, "agent_type": "ClaudeAgent"}
|
|
183
|
+
if isinstance(options.system_prompt, str):
|
|
184
|
+
config["instruction"] = options.system_prompt
|
|
185
|
+
if declarations:
|
|
186
|
+
config["tools"] = [{"function_declarations": declarations}]
|
|
187
|
+
agents = {agent_name: config}
|
|
188
|
+
subagents = options.agents or {}
|
|
189
|
+
if subagents:
|
|
190
|
+
config["sub_agents"] = list(subagents)
|
|
191
|
+
by_name = {d["name"]: d for d in declarations}
|
|
192
|
+
for name, definition in subagents.items():
|
|
193
|
+
sub: dict[str, Any] = {
|
|
194
|
+
"agent_id": name,
|
|
195
|
+
"agent_type": "ClaudeAgent",
|
|
196
|
+
"description": definition.description,
|
|
197
|
+
"instruction": definition.prompt,
|
|
198
|
+
}
|
|
199
|
+
# A subagent without its own tool list gets all of the agent's tools.
|
|
200
|
+
tools = (
|
|
201
|
+
declarations
|
|
202
|
+
if definition.tools is None
|
|
203
|
+
else [by_name.get(tool, {"name": tool}) for tool in definition.tools]
|
|
204
|
+
)
|
|
205
|
+
if tools:
|
|
206
|
+
sub["tools"] = [{"function_declarations": tools}]
|
|
207
|
+
agents[name] = sub
|
|
208
|
+
return agents
|
|
209
|
+
|
|
210
|
+
|
|
211
|
+
def _fill_case(
|
|
212
|
+
case: dict,
|
|
213
|
+
agent_name: str,
|
|
214
|
+
infer: Callable[[str, str], tuple[str, list[dict[str, Any]]]],
|
|
215
|
+
agents: dict | None,
|
|
216
|
+
) -> None:
|
|
217
|
+
"""Run one case through the agent and write its replies into it."""
|
|
218
|
+
agent_data = case.setdefault("agent_data", {})
|
|
219
|
+
if agents:
|
|
220
|
+
agent_data.setdefault("agents", agents)
|
|
221
|
+
turns = agent_data.setdefault("turns", [])
|
|
222
|
+
reply: str | None = None
|
|
223
|
+
conversation_id = uuid.uuid4().hex
|
|
224
|
+
|
|
225
|
+
for turn in turns:
|
|
226
|
+
prompt = _user_prompt(turn)
|
|
227
|
+
if prompt is None:
|
|
228
|
+
continue
|
|
229
|
+
reply, tool_calls = infer(prompt, conversation_id)
|
|
230
|
+
turn.setdefault("events", []).extend(
|
|
231
|
+
_agent_events(agent_name, reply, tool_calls)
|
|
232
|
+
)
|
|
233
|
+
|
|
234
|
+
# A prompt-shaped case has no turns: the reply becomes turn 0.
|
|
235
|
+
if not turns:
|
|
236
|
+
prompt = _prompt_text(case.get("prompt"))
|
|
237
|
+
if prompt is not None:
|
|
238
|
+
reply, tool_calls = infer(prompt, conversation_id)
|
|
239
|
+
turns.append({"events": _agent_events(agent_name, reply, tool_calls)})
|
|
240
|
+
|
|
241
|
+
for index, turn in enumerate(turns):
|
|
242
|
+
turn.setdefault("turn_index", index)
|
|
243
|
+
turn.setdefault("turn_id", f"turn_{index}")
|
|
244
|
+
|
|
245
|
+
if reply is not None:
|
|
246
|
+
case.setdefault("responses", []).append(
|
|
247
|
+
{"response": {"role": "model", "parts": [{"text": reply}]}}
|
|
248
|
+
)
|
|
249
|
+
|
|
250
|
+
|
|
251
|
+
def generate_traces(
|
|
252
|
+
dataset: dict,
|
|
253
|
+
agent_name: str,
|
|
254
|
+
infer: Callable[[str, str], tuple[str, list[dict[str, Any]]]],
|
|
255
|
+
agents: dict | None = None,
|
|
256
|
+
) -> tuple[dict, list[tuple[int, str]]]:
|
|
257
|
+
"""Fill each eval case with the agent's replies.
|
|
258
|
+
|
|
259
|
+
Returns the dataset with the cases that succeeded, and `(index, error)` for
|
|
260
|
+
each case that failed. `eval grade` rejects a trace whose turns lack
|
|
261
|
+
`turn_index`/`turn_id` and scores nothing without `responses`.
|
|
262
|
+
"""
|
|
263
|
+
succeeded: list[dict] = []
|
|
264
|
+
failures: list[tuple[int, str]] = []
|
|
265
|
+
for index, case in enumerate(dataset.get("eval_cases") or []):
|
|
266
|
+
try:
|
|
267
|
+
_fill_case(case, agent_name, infer, agents)
|
|
268
|
+
except Exception as e:
|
|
269
|
+
logging.warning("Eval case %d failed: %s", index, e)
|
|
270
|
+
failures.append((index, f"{type(e).__name__}: {e}"))
|
|
271
|
+
continue
|
|
272
|
+
succeeded.append(case)
|
|
273
|
+
return {**dataset, "eval_cases": succeeded}, failures
|
|
274
|
+
|
|
275
|
+
|
|
276
|
+
def main() -> int:
|
|
277
|
+
parser = argparse.ArgumentParser(description=__doc__)
|
|
278
|
+
parser.add_argument("--dataset", default="tests/eval/datasets/basic-dataset.json")
|
|
279
|
+
parser.add_argument("--output", "-o", default="artifacts/traces/")
|
|
280
|
+
# `eval run` forwards these flags, so accept them.
|
|
281
|
+
parser.add_argument("--url", default=None)
|
|
282
|
+
parser.add_argument("--app-name", default=None, help="Ignored; no server here.")
|
|
283
|
+
parser.add_argument(
|
|
284
|
+
"--concurrency", default=None, help="Ignored; cases run in order."
|
|
285
|
+
)
|
|
286
|
+
parser.add_argument(
|
|
287
|
+
"--header", action="append", default=[], help="Ignored; no server."
|
|
288
|
+
)
|
|
289
|
+
args = parser.parse_args()
|
|
290
|
+
|
|
291
|
+
if args.url:
|
|
292
|
+
# The built-in queries ADK's /run_sse, which this A2A-only server lacks.
|
|
293
|
+
print(
|
|
294
|
+
"claude: --url is not supported; this project's `eval generate` runs "
|
|
295
|
+
"the agent in process. Run it without --url to evaluate the local code.",
|
|
296
|
+
file=sys.stderr,
|
|
297
|
+
)
|
|
298
|
+
return 2
|
|
299
|
+
|
|
300
|
+
import anyio
|
|
301
|
+
from dotenv import load_dotenv
|
|
302
|
+
|
|
303
|
+
# .env first: app.agent's tool lists depend on flags such as MEMORY_BANK_ENABLED.
|
|
304
|
+
load_dotenv()
|
|
305
|
+
from app.agent import root_agent
|
|
306
|
+
from app.app_utils.claude import run_turn
|
|
307
|
+
from app.app_utils.settings import AGENT_NAME
|
|
308
|
+
|
|
309
|
+
def infer(prompt: str, conversation_id: str) -> tuple[str, list[dict[str, Any]]]:
|
|
310
|
+
reply = anyio.run(run_turn, prompt, root_agent, conversation_id)
|
|
311
|
+
return reply.text, reply.tool_calls + reply.subagent_calls
|
|
312
|
+
|
|
313
|
+
raw = json.loads(Path(args.dataset).read_text(encoding="utf-8"))
|
|
314
|
+
out, failures = generate_traces(
|
|
315
|
+
raw, AGENT_NAME, infer, agents_metadata(root_agent, AGENT_NAME)
|
|
316
|
+
)
|
|
317
|
+
n_cases = len(raw.get("eval_cases") or [])
|
|
318
|
+
n_succeeded = n_cases - len(failures)
|
|
319
|
+
if failures:
|
|
320
|
+
print(
|
|
321
|
+
f"\nInference summary: {n_succeeded}/{n_cases} succeeded, "
|
|
322
|
+
f"{len(failures)} failed.",
|
|
323
|
+
file=sys.stderr,
|
|
324
|
+
)
|
|
325
|
+
for index, error in failures:
|
|
326
|
+
print(f" case {index}: {error}", file=sys.stderr)
|
|
327
|
+
if n_cases and not n_succeeded:
|
|
328
|
+
print(
|
|
329
|
+
f"No artifact written: 0 of {n_cases} cases produced output.",
|
|
330
|
+
file=sys.stderr,
|
|
331
|
+
)
|
|
332
|
+
return 1
|
|
333
|
+
out_path = _resolve_output_path(args.output)
|
|
334
|
+
out_path.write_text(json.dumps(out, indent=2), encoding="utf-8")
|
|
335
|
+
print(f"[claude-generate] wrote {out_path}")
|
|
336
|
+
return 0
|
|
337
|
+
|
|
338
|
+
|
|
339
|
+
if __name__ == "__main__":
|
|
340
|
+
sys.exit(main())
|
|
@@ -0,0 +1,66 @@
|
|
|
1
|
+
# Copyright 2026 Google LLC
|
|
2
|
+
#
|
|
3
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
4
|
+
# you may not use this file except in compliance with the License.
|
|
5
|
+
# You may obtain a copy of the License at
|
|
6
|
+
#
|
|
7
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
|
8
|
+
#
|
|
9
|
+
# Unless required by applicable law or agreed to in writing, software
|
|
10
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
11
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
12
|
+
# See the License for the specific language governing permissions and
|
|
13
|
+
# limitations under the License.
|
|
14
|
+
|
|
15
|
+
"""Register with Gemini over A2A; refuse on Agent Runtime.
|
|
16
|
+
|
|
17
|
+
Registration on Agent Runtime is not supported for Claude agents yet.
|
|
18
|
+
"""
|
|
19
|
+
|
|
20
|
+
from __future__ import annotations
|
|
21
|
+
|
|
22
|
+
import json
|
|
23
|
+
import os
|
|
24
|
+
import pathlib
|
|
25
|
+
import subprocess
|
|
26
|
+
import sys
|
|
27
|
+
|
|
28
|
+
import yaml
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
def _deployed_to_agent_runtime() -> bool:
|
|
32
|
+
targets = []
|
|
33
|
+
manifest = pathlib.Path("agents-cli-manifest.yaml")
|
|
34
|
+
if manifest.exists():
|
|
35
|
+
data = yaml.safe_load(manifest.read_text(encoding="utf-8"))
|
|
36
|
+
create_params = data.get("create_params") if isinstance(data, dict) else None
|
|
37
|
+
if isinstance(create_params, dict):
|
|
38
|
+
targets.append(create_params.get("deployment_target"))
|
|
39
|
+
# `agents-cli deploy --deployment-target` can differ from the manifest.
|
|
40
|
+
metadata = pathlib.Path("deployment_metadata.json")
|
|
41
|
+
if metadata.exists():
|
|
42
|
+
data = json.loads(metadata.read_text(encoding="utf-8"))
|
|
43
|
+
if isinstance(data, dict):
|
|
44
|
+
targets.append(data.get("deployment_target"))
|
|
45
|
+
return "agent_runtime" in targets
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
def main() -> int:
|
|
49
|
+
if _deployed_to_agent_runtime():
|
|
50
|
+
print(
|
|
51
|
+
"Publishing to Gemini is not supported yet for Claude agents on "
|
|
52
|
+
"Agent Runtime. Deploy to cloud_run and publish from there, where "
|
|
53
|
+
"registration is A2A.",
|
|
54
|
+
file=sys.stderr,
|
|
55
|
+
)
|
|
56
|
+
return 2
|
|
57
|
+
|
|
58
|
+
env = {**os.environ, "AGENTS_CLI_DISABLE_OVERRIDES": "1"}
|
|
59
|
+
# The pre-rename name, which every agents-cli version accepts.
|
|
60
|
+
return subprocess.run(
|
|
61
|
+
["agents-cli", "publish", "gemini-enterprise", *sys.argv[1:]], env=env
|
|
62
|
+
).returncode
|
|
63
|
+
|
|
64
|
+
|
|
65
|
+
if __name__ == "__main__":
|
|
66
|
+
sys.exit(main())
|