google-agents-cli-extensions 1.14.2__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- core/claude/README.md +35 -0
- core/claude/template/.agents/skills/agents-cli-claude/SKILL.md +106 -0
- core/claude/template/.agents/skills/agents-cli-claude/references/claude.md +386 -0
- core/claude/template/.agents/skills/agents-cli-claude/references/migrate-claude-agent.md +318 -0
- core/claude/template/.agents/skills/agents-cli-claude/references/migrate-from-aws.md +132 -0
- core/claude/template/.dockerignore +28 -0
- core/claude/template/.env +18 -0
- core/claude/template/.env.example +17 -0
- core/claude/template/.template/templateconfig.yaml +31 -0
- core/claude/template/AGENTS.md +33 -0
- core/claude/template/Dockerfile +55 -0
- core/claude/template/agent_config/README.md +9 -0
- core/claude/template/agents-cli-extension.yaml +49 -0
- core/claude/template/app/__init__.py +15 -0
- core/claude/template/app/agent.py +49 -0
- core/claude/template/app/app_utils/a2a.py +245 -0
- core/claude/template/app/app_utils/bq_analytics/__init__.py +124 -0
- core/claude/template/app/app_utils/bq_analytics/recorder.py +593 -0
- core/claude/template/app/app_utils/bq_analytics/redaction.py +209 -0
- core/claude/template/app/app_utils/bq_analytics/schema.py +394 -0
- core/claude/template/app/app_utils/bq_analytics/tables.py +104 -0
- core/claude/template/app/app_utils/bq_analytics/uploader.py +188 -0
- core/claude/template/app/app_utils/claude.py +297 -0
- core/claude/template/app/app_utils/completions.py +225 -0
- core/claude/template/app/app_utils/conversation.py +29 -0
- core/claude/template/app/app_utils/memory_bank.py +308 -0
- core/claude/template/app/app_utils/project_settings.py +91 -0
- core/claude/template/app/app_utils/reasoning_engine.py +237 -0
- core/claude/template/app/app_utils/remote_agents.py +349 -0
- core/claude/template/app/app_utils/remote_auth.py +159 -0
- core/claude/template/app/app_utils/replies.py +143 -0
- core/claude/template/app/app_utils/session_store.py +174 -0
- core/claude/template/app/app_utils/settings.py +24 -0
- core/claude/template/app/app_utils/telemetry/__init__.py +114 -0
- core/claude/template/app/app_utils/telemetry/call_counts.py +44 -0
- core/claude/template/app/app_utils/telemetry/cli_relay.py +266 -0
- core/claude/template/app/app_utils/telemetry/config.py +133 -0
- core/claude/template/app/app_utils/telemetry/logs.py +175 -0
- core/claude/template/app/app_utils/telemetry/metrics.py +270 -0
- core/claude/template/app/app_utils/telemetry/request_flush.py +104 -0
- core/claude/template/app/app_utils/telemetry/subagent_spans.py +80 -0
- core/claude/template/app/app_utils/telemetry/traces.py +166 -0
- core/claude/template/app/app_utils/turn_events.py +463 -0
- core/claude/template/app/app_utils/turn_options.py +219 -0
- core/claude/template/app/app_utils/workspaces.py +65 -0
- core/claude/template/app/fast_api_app.py +71 -0
- core/claude/template/app/feature_tools.py +27 -0
- core/claude/template/pyproject.toml +100 -0
- core/claude/template/scripts/eval_generate.py +340 -0
- core/claude/template/scripts/publish.py +66 -0
- core/claude/template/scripts/run.py +85 -0
- core/claude/template/scripts/unsupported.py +48 -0
- core/claude/template/tests/eval/datasets/basic-dataset.json +31 -0
- core/claude/template/tests/eval/eval_config.yaml +17 -0
- core/claude/template/tests/eval/response_quality.py +60 -0
- core/claude/template/tests/integration/test_server_e2e.py +168 -0
- core/claude/template/tests/load_test/.results/.placeholder +0 -0
- core/claude/template/tests/load_test/README.md +24 -0
- core/claude/template/tests/load_test/load_test.py +118 -0
- core/claude/template/tests/unit/conftest.py +63 -0
- core/claude/template/tests/unit/helpers.py +32 -0
- core/claude/template/tests/unit/test_a2a.py +432 -0
- core/claude/template/tests/unit/test_bq_analytics.py +928 -0
- core/claude/template/tests/unit/test_bq_redaction.py +89 -0
- core/claude/template/tests/unit/test_claude.py +825 -0
- core/claude/template/tests/unit/test_completions.py +369 -0
- core/claude/template/tests/unit/test_fast_api_app.py +59 -0
- core/claude/template/tests/unit/test_feature_tools.py +55 -0
- core/claude/template/tests/unit/test_memory_bank.py +625 -0
- core/claude/template/tests/unit/test_reasoning_engine.py +295 -0
- core/claude/template/tests/unit/test_remote_agents.py +517 -0
- core/claude/template/tests/unit/test_telemetry.py +991 -0
- core/claude/template/tests/unit/test_turn_events.py +309 -0
- core/claude/template/tests/unit/test_turn_options.py +394 -0
- core/claude/template/uv.lock +3029 -0
- core/langchain/README.md +54 -0
- core/langchain/template/.agents/skills/agents-cli-langchain/SKILL.md +84 -0
- core/langchain/template/.agents/skills/agents-cli-langchain/references/langchain.md +120 -0
- core/langchain/template/.agents/skills/agents-cli-langchain/references/samples.md +42 -0
- core/langchain/template/.template/templateconfig.yaml +32 -0
- core/langchain/template/AGENTS.md +35 -0
- core/langchain/template/agents-cli-extension.yaml +54 -0
- core/langchain/template/app/__init__.py +19 -0
- core/langchain/template/app/agent.py +61 -0
- core/langchain/template/app/app_utils/a2a.py +176 -0
- core/langchain/template/app/app_utils/content.py +37 -0
- core/langchain/template/app/app_utils/telemetry.py +154 -0
- core/langchain/template/app/fast_api_app.py +68 -0
- core/langchain/template/langgraph.json +7 -0
- core/langchain/template/pyproject.toml +102 -0
- core/langchain/template/scripts/_content.py +40 -0
- core/langchain/template/scripts/eval_generate.py +202 -0
- core/langchain/template/scripts/publish.py +59 -0
- core/langchain/template/scripts/run.py +59 -0
- core/langchain/template/scripts/unsupported.py +48 -0
- core/langchain/template/tests/eval/datasets/basic-dataset.json +16 -0
- core/langchain/template/tests/eval/eval_config.yaml +32 -0
- core/langchain/template/tests/eval/response_quality.py +60 -0
- core/langchain/template/tests/integration/test_server_e2e.py +173 -0
- core/langchain/template/tests/load_test/.results/.placeholder +0 -0
- core/langchain/template/tests/load_test/README.md +20 -0
- core/langchain/template/tests/load_test/load_test.py +82 -0
- core/langchain/template/uv.lock +3820 -0
- google_agents_cli_extensions-1.14.2.dist-info/METADATA +133 -0
- google_agents_cli_extensions-1.14.2.dist-info/RECORD +107 -0
- google_agents_cli_extensions-1.14.2.dist-info/WHEEL +4 -0
- google_agents_cli_extensions-1.14.2.dist-info/licenses/LICENSE +201 -0
|
@@ -0,0 +1,85 @@
|
|
|
1
|
+
# Copyright 2026 Google LLC
|
|
2
|
+
#
|
|
3
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
4
|
+
# you may not use this file except in compliance with the License.
|
|
5
|
+
# You may obtain a copy of the License at
|
|
6
|
+
#
|
|
7
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
|
8
|
+
#
|
|
9
|
+
# Unless required by applicable law or agreed to in writing, software
|
|
10
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
11
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
12
|
+
# See the License for the specific language governing permissions and
|
|
13
|
+
# limitations under the License.
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
"""Run `root_agent` in process for `agents-cli run`.
|
|
17
|
+
|
|
18
|
+
With `--url` or `--mode`, the built-in `agents-cli run` queries the served agent.
|
|
19
|
+
"""
|
|
20
|
+
|
|
21
|
+
from __future__ import annotations
|
|
22
|
+
|
|
23
|
+
import argparse
|
|
24
|
+
import json
|
|
25
|
+
import os
|
|
26
|
+
import subprocess
|
|
27
|
+
import sys
|
|
28
|
+
import uuid
|
|
29
|
+
from pathlib import Path
|
|
30
|
+
|
|
31
|
+
# `app` may not be installed as a package, so import it from the project root.
|
|
32
|
+
sys.path.insert(0, str(Path(__file__).resolve().parent.parent))
|
|
33
|
+
|
|
34
|
+
_REMOTE_FLAGS = ("--url", "--mode")
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
def main() -> int:
|
|
38
|
+
argv = sys.argv[1:]
|
|
39
|
+
if any(arg.split("=", 1)[0] in _REMOTE_FLAGS for arg in argv):
|
|
40
|
+
env = {**os.environ, "AGENTS_CLI_DISABLE_OVERRIDES": "1"}
|
|
41
|
+
return subprocess.run(["agents-cli", "run", *argv], env=env).returncode
|
|
42
|
+
|
|
43
|
+
parser = argparse.ArgumentParser(prog="agents-cli run", description=__doc__)
|
|
44
|
+
parser.add_argument("message", help="The prompt to send to the agent.")
|
|
45
|
+
parser.add_argument(
|
|
46
|
+
"--session-id",
|
|
47
|
+
default=None,
|
|
48
|
+
help="Continue this conversation (printed after each run).",
|
|
49
|
+
)
|
|
50
|
+
parser.add_argument(
|
|
51
|
+
"-v",
|
|
52
|
+
"--verbose",
|
|
53
|
+
action="store_true",
|
|
54
|
+
help="Print each tool call (name, arguments and result) as JSON.",
|
|
55
|
+
)
|
|
56
|
+
args = parser.parse_args(argv)
|
|
57
|
+
|
|
58
|
+
import anyio
|
|
59
|
+
from claude_agent_sdk import ClaudeSDKError
|
|
60
|
+
from dotenv import load_dotenv
|
|
61
|
+
|
|
62
|
+
# .env first: app.agent's tool lists depend on flags such as MEMORY_BANK_ENABLED.
|
|
63
|
+
load_dotenv()
|
|
64
|
+
from app.agent import root_agent
|
|
65
|
+
from app.app_utils.claude import AgentError, run_turn
|
|
66
|
+
|
|
67
|
+
conversation_id = args.session_id or uuid.uuid4().hex
|
|
68
|
+
try:
|
|
69
|
+
reply = anyio.run(run_turn, args.message, root_agent, conversation_id)
|
|
70
|
+
except (AgentError, ClaudeSDKError) as e:
|
|
71
|
+
print(e, file=sys.stderr)
|
|
72
|
+
return 1
|
|
73
|
+
if args.verbose:
|
|
74
|
+
for call in reply.tool_calls:
|
|
75
|
+
print(json.dumps(call, indent=2, default=str), file=sys.stderr)
|
|
76
|
+
print(reply.text)
|
|
77
|
+
print(
|
|
78
|
+
f"\nSession: {conversation_id}\n"
|
|
79
|
+
f' Resume with: agents-cli run "<message>" --session-id {conversation_id}'
|
|
80
|
+
)
|
|
81
|
+
return 0
|
|
82
|
+
|
|
83
|
+
|
|
84
|
+
if __name__ == "__main__":
|
|
85
|
+
sys.exit(main())
|
|
@@ -0,0 +1,48 @@
|
|
|
1
|
+
# Copyright 2026 Google LLC
|
|
2
|
+
#
|
|
3
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
4
|
+
# you may not use this file except in compliance with the License.
|
|
5
|
+
# You may obtain a copy of the License at
|
|
6
|
+
#
|
|
7
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
|
8
|
+
#
|
|
9
|
+
# Unless required by applicable law or agreed to in writing, software
|
|
10
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
11
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
12
|
+
# See the License for the specific language governing permissions and
|
|
13
|
+
# limitations under the License.
|
|
14
|
+
|
|
15
|
+
"""Explain why an ADK-only agents-cli command does not work for this project."""
|
|
16
|
+
|
|
17
|
+
from __future__ import annotations
|
|
18
|
+
|
|
19
|
+
import sys
|
|
20
|
+
|
|
21
|
+
_REASONS = {
|
|
22
|
+
"eval dataset": (
|
|
23
|
+
"it runs the agent through ADK's user simulator, which needs google-adk "
|
|
24
|
+
"in this project's venv."
|
|
25
|
+
),
|
|
26
|
+
"eval optimize": "it shells out to the `adk` binary, which this project does not use.",
|
|
27
|
+
"playground": (
|
|
28
|
+
'the Claude Agent SDK has no local dev UI. Use `agents-cli run "<prompt>"`, '
|
|
29
|
+
"or serve the agent locally with `uv run uvicorn app.fast_api_app:app "
|
|
30
|
+
"--port 8080` and query it over A2A."
|
|
31
|
+
),
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
def main() -> int:
|
|
36
|
+
command = sys.argv[1] if len(sys.argv) > 1 else "this command"
|
|
37
|
+
reason = _REASONS.get(
|
|
38
|
+
command, "it depends on ADK, which this project does not use."
|
|
39
|
+
)
|
|
40
|
+
print(
|
|
41
|
+
f"agents-cli {command} is not supported by the claude extension: {reason}",
|
|
42
|
+
file=sys.stderr,
|
|
43
|
+
)
|
|
44
|
+
return 2
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
if __name__ == "__main__":
|
|
48
|
+
sys.exit(main())
|
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
{
|
|
2
|
+
"eval_cases": [
|
|
3
|
+
{
|
|
4
|
+
"name": "greeting",
|
|
5
|
+
"agent_data": {
|
|
6
|
+
"turns": [
|
|
7
|
+
{
|
|
8
|
+
"events": [
|
|
9
|
+
{"author": "user", "content": {"parts": [{"text": "hello"}]}}
|
|
10
|
+
]
|
|
11
|
+
}
|
|
12
|
+
]
|
|
13
|
+
}
|
|
14
|
+
},
|
|
15
|
+
{
|
|
16
|
+
"name": "weather_lookup",
|
|
17
|
+
"agent_data": {
|
|
18
|
+
"turns": [
|
|
19
|
+
{
|
|
20
|
+
"events": [
|
|
21
|
+
{
|
|
22
|
+
"author": "user",
|
|
23
|
+
"content": {"parts": [{"text": "What's the weather in San Francisco?"}]}
|
|
24
|
+
}
|
|
25
|
+
]
|
|
26
|
+
}
|
|
27
|
+
]
|
|
28
|
+
}
|
|
29
|
+
}
|
|
30
|
+
]
|
|
31
|
+
}
|
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
metrics_to_run:
|
|
2
|
+
- custom_response_quality
|
|
3
|
+
|
|
4
|
+
# Fail `agents-cli eval run` (exit 1) when a metric's mean score drops below
|
|
5
|
+
# its threshold. Set it once you know the bar this agent must meet.
|
|
6
|
+
# thresholds:
|
|
7
|
+
# custom_response_quality: 4 # the judge scores 1-5
|
|
8
|
+
|
|
9
|
+
custom_metrics:
|
|
10
|
+
# Local LLM judge via google-genai: ADC on Gemini platform, or GEMINI_API_KEY.
|
|
11
|
+
- name: custom_response_quality
|
|
12
|
+
custom_function_file: response_quality.py
|
|
13
|
+
- name: agent_turn_count
|
|
14
|
+
custom_function: |
|
|
15
|
+
def evaluate(instance):
|
|
16
|
+
turns = (instance.get("agent_data") or {}).get("turns", [])
|
|
17
|
+
return {'score': len(turns)}
|
|
@@ -0,0 +1,60 @@
|
|
|
1
|
+
# Copyright 2026 Google LLC
|
|
2
|
+
#
|
|
3
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
4
|
+
# you may not use this file except in compliance with the License.
|
|
5
|
+
# You may obtain a copy of the License at
|
|
6
|
+
#
|
|
7
|
+
# https://www.apache.org/licenses/LICENSE-2.0
|
|
8
|
+
#
|
|
9
|
+
# Unless required by applicable law or agreed to in writing, software
|
|
10
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
11
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
12
|
+
# See the License for the specific language governing permissions and
|
|
13
|
+
# limitations under the License.
|
|
14
|
+
|
|
15
|
+
"""Local LLM-as-judge for `custom_response_quality` (see eval_config.yaml)."""
|
|
16
|
+
|
|
17
|
+
from google import genai
|
|
18
|
+
from google.genai import types
|
|
19
|
+
from pydantic import BaseModel
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
class _Verdict(BaseModel):
|
|
23
|
+
score: int # 1-5
|
|
24
|
+
explanation: str
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
def evaluate(instance):
|
|
28
|
+
reference = instance.get("reference")
|
|
29
|
+
rubric = (
|
|
30
|
+
"Grade the agent's final response on a 1-5 scale (1 poor, 5 excellent) for "
|
|
31
|
+
"accuracy, relevance, and clarity."
|
|
32
|
+
)
|
|
33
|
+
if reference:
|
|
34
|
+
rubric += (
|
|
35
|
+
" The response should agree with the expected answer below; penalize "
|
|
36
|
+
"factual disagreement with it."
|
|
37
|
+
)
|
|
38
|
+
prompt = (
|
|
39
|
+
f"You are an expert QA evaluator for an enterprise AI assistant. {rubric}\n"
|
|
40
|
+
f"User Prompt: {instance.get('prompt', '')}\n"
|
|
41
|
+
f"Final Response: {instance.get('response', '')}\n"
|
|
42
|
+
)
|
|
43
|
+
if reference:
|
|
44
|
+
prompt += f"Expected Answer (ground truth): {reference}\n"
|
|
45
|
+
prompt += f"Full Agent Trace: {instance.get('agent_data', '')}\n"
|
|
46
|
+
|
|
47
|
+
client = genai.Client() # AI Studio (GEMINI_API_KEY) or Gemini platform (ADC)
|
|
48
|
+
response = client.models.generate_content(
|
|
49
|
+
model="gemini-3.8-flash",
|
|
50
|
+
contents=prompt,
|
|
51
|
+
config=types.GenerateContentConfig(
|
|
52
|
+
temperature=0,
|
|
53
|
+
response_mime_type="application/json",
|
|
54
|
+
response_schema=_Verdict,
|
|
55
|
+
),
|
|
56
|
+
)
|
|
57
|
+
verdict = response.parsed
|
|
58
|
+
if verdict is None:
|
|
59
|
+
return {"score": 0, "explanation": response.text or ""}
|
|
60
|
+
return {"score": max(1, min(5, verdict.score)), "explanation": verdict.explanation}
|
|
@@ -0,0 +1,168 @@
|
|
|
1
|
+
# Copyright 2026 Google LLC
|
|
2
|
+
#
|
|
3
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
4
|
+
# you may not use this file except in compliance with the License.
|
|
5
|
+
# You may obtain a copy of the License at
|
|
6
|
+
#
|
|
7
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
|
8
|
+
#
|
|
9
|
+
# Unless required by applicable law or agreed to in writing, software
|
|
10
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
11
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
12
|
+
# See the License for the specific language governing permissions and
|
|
13
|
+
# limitations under the License.
|
|
14
|
+
|
|
15
|
+
"""End-to-end tests against the A2A server."""
|
|
16
|
+
|
|
17
|
+
import asyncio
|
|
18
|
+
import logging
|
|
19
|
+
import os
|
|
20
|
+
import subprocess
|
|
21
|
+
import sys
|
|
22
|
+
import threading
|
|
23
|
+
import time
|
|
24
|
+
import uuid
|
|
25
|
+
from collections.abc import Iterator
|
|
26
|
+
from typing import Any
|
|
27
|
+
|
|
28
|
+
import httpx
|
|
29
|
+
import pytest
|
|
30
|
+
import requests
|
|
31
|
+
from a2a.client import ClientConfig, create_client
|
|
32
|
+
from a2a.types import (
|
|
33
|
+
Message,
|
|
34
|
+
Part,
|
|
35
|
+
Role,
|
|
36
|
+
SendMessageRequest,
|
|
37
|
+
StreamResponse,
|
|
38
|
+
TaskState,
|
|
39
|
+
)
|
|
40
|
+
from requests.exceptions import RequestException
|
|
41
|
+
|
|
42
|
+
logging.basicConfig(level=logging.INFO)
|
|
43
|
+
logger = logging.getLogger(__name__)
|
|
44
|
+
|
|
45
|
+
BASE_URL = "http://127.0.0.1:8000"
|
|
46
|
+
A2A_RPC_URL = BASE_URL + "/a2a/app/"
|
|
47
|
+
AGENT_CARD_URL = A2A_RPC_URL + ".well-known/agent-card.json"
|
|
48
|
+
|
|
49
|
+
|
|
50
|
+
def log_output(pipe: Any, log_func: Any) -> None:
|
|
51
|
+
for line in iter(pipe.readline, ""):
|
|
52
|
+
log_func(line.strip())
|
|
53
|
+
|
|
54
|
+
|
|
55
|
+
def start_server() -> subprocess.Popen[str]:
|
|
56
|
+
command = [
|
|
57
|
+
sys.executable,
|
|
58
|
+
"-m",
|
|
59
|
+
"uvicorn",
|
|
60
|
+
"app.fast_api_app:app",
|
|
61
|
+
"--host",
|
|
62
|
+
"0.0.0.0",
|
|
63
|
+
"--port",
|
|
64
|
+
"8000",
|
|
65
|
+
]
|
|
66
|
+
env = os.environ.copy()
|
|
67
|
+
env["INTEGRATION_TEST"] = "TRUE"
|
|
68
|
+
# The agent card advertises this, and the A2A client follows it.
|
|
69
|
+
env["APP_URL"] = BASE_URL
|
|
70
|
+
process = subprocess.Popen(
|
|
71
|
+
command,
|
|
72
|
+
stdout=subprocess.PIPE,
|
|
73
|
+
stderr=subprocess.PIPE,
|
|
74
|
+
text=True,
|
|
75
|
+
bufsize=1,
|
|
76
|
+
env=env,
|
|
77
|
+
)
|
|
78
|
+
threading.Thread(
|
|
79
|
+
target=log_output, args=(process.stdout, logger.info), daemon=True
|
|
80
|
+
).start()
|
|
81
|
+
threading.Thread(
|
|
82
|
+
target=log_output, args=(process.stderr, logger.error), daemon=True
|
|
83
|
+
).start()
|
|
84
|
+
return process
|
|
85
|
+
|
|
86
|
+
|
|
87
|
+
def wait_for_server(timeout: int = 90, interval: int = 1) -> bool:
|
|
88
|
+
"""The card is only served once the lifespan has mounted the A2A routes."""
|
|
89
|
+
start_time = time.time()
|
|
90
|
+
while time.time() - start_time < timeout:
|
|
91
|
+
try:
|
|
92
|
+
if requests.get(AGENT_CARD_URL, timeout=10).status_code == 200:
|
|
93
|
+
logger.info("Server is ready")
|
|
94
|
+
return True
|
|
95
|
+
except RequestException:
|
|
96
|
+
pass
|
|
97
|
+
time.sleep(interval)
|
|
98
|
+
logger.error(f"Server did not become ready within {timeout} seconds")
|
|
99
|
+
return False
|
|
100
|
+
|
|
101
|
+
|
|
102
|
+
@pytest.fixture(scope="session")
|
|
103
|
+
def server_fixture(request: Any) -> Iterator[subprocess.Popen[str]]:
|
|
104
|
+
logger.info("Starting server process")
|
|
105
|
+
server_process = start_server()
|
|
106
|
+
if not wait_for_server():
|
|
107
|
+
pytest.fail("Server failed to start")
|
|
108
|
+
|
|
109
|
+
def stop_server() -> None:
|
|
110
|
+
server_process.terminate()
|
|
111
|
+
server_process.wait()
|
|
112
|
+
|
|
113
|
+
request.addfinalizer(stop_server)
|
|
114
|
+
yield server_process
|
|
115
|
+
|
|
116
|
+
|
|
117
|
+
def test_a2a_chat_stream(server_fixture: subprocess.Popen[str]) -> None:
|
|
118
|
+
|
|
119
|
+
async def _stream() -> list[StreamResponse]:
|
|
120
|
+
config = ClientConfig(
|
|
121
|
+
streaming=True, httpx_client=httpx.AsyncClient(timeout=60.0)
|
|
122
|
+
)
|
|
123
|
+
client = await create_client(A2A_RPC_URL.rstrip("/"), config)
|
|
124
|
+
message = Message(
|
|
125
|
+
message_id=f"msg-user-{uuid.uuid4()}",
|
|
126
|
+
role=Role.ROLE_USER,
|
|
127
|
+
parts=[Part(text="Hi!")],
|
|
128
|
+
)
|
|
129
|
+
return [
|
|
130
|
+
chunk
|
|
131
|
+
async for chunk in client.send_message(SendMessageRequest(message=message))
|
|
132
|
+
]
|
|
133
|
+
|
|
134
|
+
responses = asyncio.run(_stream())
|
|
135
|
+
assert responses, "No responses received from stream"
|
|
136
|
+
|
|
137
|
+
def _is_completed(chunk: StreamResponse) -> bool:
|
|
138
|
+
if chunk.HasField("status_update"):
|
|
139
|
+
return chunk.status_update.status.state == TaskState.TASK_STATE_COMPLETED
|
|
140
|
+
if chunk.HasField("task"):
|
|
141
|
+
return chunk.task.status.state == TaskState.TASK_STATE_COMPLETED
|
|
142
|
+
return False
|
|
143
|
+
|
|
144
|
+
assert any(_is_completed(chunk) for chunk in responses), (
|
|
145
|
+
"No completed task received from stream"
|
|
146
|
+
)
|
|
147
|
+
|
|
148
|
+
|
|
149
|
+
def test_agent_card(server_fixture: subprocess.Popen[str]) -> None:
|
|
150
|
+
"""Deploy and `agents-cli run --mode a2a` both read this card."""
|
|
151
|
+
response = requests.get(AGENT_CARD_URL, timeout=10)
|
|
152
|
+
assert response.status_code == 200, f"A2A endpoint returned {response.status_code}"
|
|
153
|
+
|
|
154
|
+
served_agent_card = response.json()
|
|
155
|
+
for field in (
|
|
156
|
+
"name",
|
|
157
|
+
"description",
|
|
158
|
+
"skills",
|
|
159
|
+
"capabilities",
|
|
160
|
+
"version",
|
|
161
|
+
"supportedInterfaces",
|
|
162
|
+
):
|
|
163
|
+
assert field in served_agent_card, f"Missing field in agent card: {field}"
|
|
164
|
+
|
|
165
|
+
|
|
166
|
+
def test_health(server_fixture: subprocess.Popen[str]) -> None:
|
|
167
|
+
"""Cloud Run probes this."""
|
|
168
|
+
assert requests.get(f"{BASE_URL}/health", timeout=10).status_code == 200
|
|
File without changes
|
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
# Load testing
|
|
2
|
+
|
|
3
|
+
[Locust](http://locust.io) drives the A2A `message/stream` route this project serves.
|
|
4
|
+
|
|
5
|
+
Local, against `uv run uvicorn app.fast_api_app:app --port 8000`:
|
|
6
|
+
|
|
7
|
+
```bash
|
|
8
|
+
uv run --with locust locust -f tests/load_test/load_test.py -H http://127.0.0.1:8000 \
|
|
9
|
+
-u 10 -r 2 -t 30s --headless
|
|
10
|
+
```
|
|
11
|
+
|
|
12
|
+
Against a deployed service, send an identity token and point at its URL:
|
|
13
|
+
|
|
14
|
+
```bash
|
|
15
|
+
export _ID_TOKEN=$(gcloud auth print-identity-token -q)
|
|
16
|
+
uv run --with locust locust -f tests/load_test/load_test.py -H "$RUN_SERVICE_URL" \
|
|
17
|
+
-u 10 -r 2 -t 60s --headless --csv=.results/report
|
|
18
|
+
```
|
|
19
|
+
|
|
20
|
+
On Agent Runtime, run it from the project root after `agents-cli deploy`, with
|
|
21
|
+
`_AUTH_TOKEN=$(gcloud auth print-access-token -q)` and no `-H`: it reads the agent from
|
|
22
|
+
`deployment_metadata.json`.
|
|
23
|
+
|
|
24
|
+
`--csv` writes the latency and failure tables Locust prints at the end.
|
|
@@ -0,0 +1,118 @@
|
|
|
1
|
+
# Copyright 2026 Google LLC
|
|
2
|
+
#
|
|
3
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
4
|
+
# you may not use this file except in compliance with the License.
|
|
5
|
+
# You may obtain a copy of the License at
|
|
6
|
+
#
|
|
7
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
|
8
|
+
#
|
|
9
|
+
# Unless required by applicable law or agreed to in writing, software
|
|
10
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
11
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
12
|
+
# See the License for the specific language governing permissions and
|
|
13
|
+
# limitations under the License.
|
|
14
|
+
|
|
15
|
+
"""Load test against the A2A route."""
|
|
16
|
+
|
|
17
|
+
import json
|
|
18
|
+
import logging
|
|
19
|
+
import os
|
|
20
|
+
import time
|
|
21
|
+
import uuid
|
|
22
|
+
|
|
23
|
+
from locust import HttpUser, between, task
|
|
24
|
+
|
|
25
|
+
ENDPOINT = "/a2a/app"
|
|
26
|
+
HOST = None
|
|
27
|
+
# Agent Runtime has no service URL, so its staging step passes no host: the A2A
|
|
28
|
+
# route sits under the engine's passthrough path.
|
|
29
|
+
if os.path.exists("deployment_metadata.json"):
|
|
30
|
+
with open("deployment_metadata.json", encoding="utf-8") as f:
|
|
31
|
+
_runtime_id = json.load(f).get("remote_agent_runtime_id")
|
|
32
|
+
if _runtime_id:
|
|
33
|
+
HOST = f"https://{_runtime_id.split('/')[3]}-aiplatform.googleapis.com"
|
|
34
|
+
ENDPOINT = f"/reasoningEngines/v1/{_runtime_id}/api/a2a/app"
|
|
35
|
+
# A2A 0.3 spells states "failed"; 1.0 spells them "TASK_STATE_FAILED".
|
|
36
|
+
_FAILED_STATES = {"failed", "rejected", "canceled"}
|
|
37
|
+
|
|
38
|
+
logging.basicConfig(
|
|
39
|
+
level=logging.INFO, format="%(asctime)s - %(name)s - %(levelname)s - %(message)s"
|
|
40
|
+
)
|
|
41
|
+
logger = logging.getLogger(__name__)
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
class ChatStreamUser(HttpUser):
|
|
45
|
+
wait_time = between(1, 3)
|
|
46
|
+
host = HOST
|
|
47
|
+
|
|
48
|
+
@task
|
|
49
|
+
def chat_stream(self) -> None:
|
|
50
|
+
headers = {"Content-Type": "application/json"}
|
|
51
|
+
# An identity token on Cloud Run, an access token on Agent Runtime.
|
|
52
|
+
token = os.environ.get("_ID_TOKEN") or os.environ.get("_AUTH_TOKEN")
|
|
53
|
+
if token:
|
|
54
|
+
headers["Authorization"] = f"Bearer {token}"
|
|
55
|
+
|
|
56
|
+
request = {
|
|
57
|
+
"jsonrpc": "2.0",
|
|
58
|
+
"id": str(uuid.uuid4()),
|
|
59
|
+
"method": "message/stream",
|
|
60
|
+
"params": {
|
|
61
|
+
"message": {
|
|
62
|
+
"messageId": f"msg-{uuid.uuid4()}",
|
|
63
|
+
"role": "user",
|
|
64
|
+
"parts": [{"text": "Hello! Weather in New York?"}],
|
|
65
|
+
}
|
|
66
|
+
},
|
|
67
|
+
}
|
|
68
|
+
|
|
69
|
+
start_time = time.time()
|
|
70
|
+
with self.client.post(
|
|
71
|
+
ENDPOINT,
|
|
72
|
+
name=f"{ENDPOINT} message/stream",
|
|
73
|
+
headers=headers,
|
|
74
|
+
json=request,
|
|
75
|
+
catch_response=True,
|
|
76
|
+
stream=True,
|
|
77
|
+
) as response:
|
|
78
|
+
if response.status_code != 200:
|
|
79
|
+
response.failure(f"Got status code {response.status_code}")
|
|
80
|
+
return
|
|
81
|
+
# A failed run still streams HTTP 200, so read the events to the end.
|
|
82
|
+
first_chunk = False
|
|
83
|
+
for line in response.iter_lines():
|
|
84
|
+
if not line:
|
|
85
|
+
continue
|
|
86
|
+
if not first_chunk:
|
|
87
|
+
first_chunk = True
|
|
88
|
+
logger.info("First chunk in %.2fs", time.time() - start_time)
|
|
89
|
+
if not line.startswith(b"data:"):
|
|
90
|
+
continue
|
|
91
|
+
try:
|
|
92
|
+
event = json.loads(line[len(b"data:") :])
|
|
93
|
+
except json.JSONDecodeError:
|
|
94
|
+
response.failure(f"Unparseable stream event: {line[:200]!r}")
|
|
95
|
+
return
|
|
96
|
+
if "error" in event:
|
|
97
|
+
response.failure(f"JSON-RPC error: {event['error']}")
|
|
98
|
+
return
|
|
99
|
+
result = event.get("result") or {}
|
|
100
|
+
if "message" in result or result.get("kind") == "message":
|
|
101
|
+
response.success()
|
|
102
|
+
return
|
|
103
|
+
state = _task_state(result)
|
|
104
|
+
if state in _FAILED_STATES:
|
|
105
|
+
response.failure(f"Task {state}")
|
|
106
|
+
return
|
|
107
|
+
if state == "completed":
|
|
108
|
+
response.success()
|
|
109
|
+
return
|
|
110
|
+
response.failure("Stream ended without a completed task")
|
|
111
|
+
|
|
112
|
+
|
|
113
|
+
def _task_state(result: dict) -> str | None:
|
|
114
|
+
"""Returns a stream event's task state in A2A 0.3 spelling, e.g. "failed"."""
|
|
115
|
+
# 1.0 wraps the event in a "statusUpdate" or "task" key; 0.3 doesn't.
|
|
116
|
+
event = result.get("statusUpdate") or result.get("task") or result
|
|
117
|
+
state = (event.get("status") or {}).get("state")
|
|
118
|
+
return state.removeprefix("TASK_STATE_").lower() if state else None
|
|
@@ -0,0 +1,63 @@
|
|
|
1
|
+
# Copyright 2026 Google LLC
|
|
2
|
+
#
|
|
3
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
4
|
+
# you may not use this file except in compliance with the License.
|
|
5
|
+
# You may obtain a copy of the License at
|
|
6
|
+
#
|
|
7
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
|
8
|
+
#
|
|
9
|
+
# Unless required by applicable law or agreed to in writing, software
|
|
10
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
11
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
12
|
+
# See the License for the specific language governing permissions and
|
|
13
|
+
# limitations under the License.
|
|
14
|
+
|
|
15
|
+
"""Fixtures shared by the tests of `app/app_utils/claude.py` and `turn_options.py`."""
|
|
16
|
+
|
|
17
|
+
import pytest
|
|
18
|
+
|
|
19
|
+
from app.app_utils import (
|
|
20
|
+
claude,
|
|
21
|
+
memory_bank,
|
|
22
|
+
project_settings,
|
|
23
|
+
turn_options,
|
|
24
|
+
workspaces,
|
|
25
|
+
)
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
@pytest.fixture
|
|
29
|
+
def isolated(monkeypatch, tmp_path):
|
|
30
|
+
"""Gemini platform env, and workspaces and project root inside tmp_path."""
|
|
31
|
+
# Keep the project's .env out of the tests: load_dotenv() finds it from
|
|
32
|
+
# claude.py's location, not cwd.
|
|
33
|
+
monkeypatch.setattr(claude, "load_dotenv", lambda: None)
|
|
34
|
+
monkeypatch.setenv("ANTHROPIC_VERTEX_PROJECT_ID", "test-project")
|
|
35
|
+
monkeypatch.delenv("ANTHROPIC_API_KEY", raising=False)
|
|
36
|
+
# Most tests run a bare ClaudeAgentOptions(), which names no model.
|
|
37
|
+
monkeypatch.setenv("ANTHROPIC_MODEL", "claude-test")
|
|
38
|
+
for name in (
|
|
39
|
+
"INSTANCE_CONNECTION_NAME",
|
|
40
|
+
"DB_PASS",
|
|
41
|
+
"DB_USER",
|
|
42
|
+
"DB_NAME",
|
|
43
|
+
"BQ_ANALYTICS_DATASET_ID",
|
|
44
|
+
"MEMORY_BANK_ENABLED",
|
|
45
|
+
"MEMORY_BANK_ENGINE_NAME",
|
|
46
|
+
"GOOGLE_CLOUD_AGENT_ENGINE_ID",
|
|
47
|
+
"ANTHROPIC_CUSTOM_HEADERS",
|
|
48
|
+
"DO_NOT_TRACK",
|
|
49
|
+
"AGENTS_CLI_TELEMETRY",
|
|
50
|
+
):
|
|
51
|
+
monkeypatch.delenv(name, raising=False)
|
|
52
|
+
monkeypatch.setenv("CLAUDE_CONFIG_DIR", str(tmp_path / "claude-config"))
|
|
53
|
+
turn_options._cloud_sql_store.cache_clear()
|
|
54
|
+
monkeypatch.setattr(memory_bank, "_engine_name", None)
|
|
55
|
+
monkeypatch.setattr(memory_bank, "_pending", set())
|
|
56
|
+
monkeypatch.setattr(workspaces, "WORKSPACES", tmp_path / "workspaces")
|
|
57
|
+
monkeypatch.setattr(workspaces, "_workspaces", {})
|
|
58
|
+
project = tmp_path / "project"
|
|
59
|
+
agent_config = project / "agent_config"
|
|
60
|
+
agent_config.mkdir(parents=True)
|
|
61
|
+
monkeypatch.setattr(project_settings, "PROJECT_ROOT", project)
|
|
62
|
+
monkeypatch.setattr(project_settings, "AGENT_CONFIG_DIR", agent_config)
|
|
63
|
+
return agent_config
|
|
@@ -0,0 +1,32 @@
|
|
|
1
|
+
# Copyright 2026 Google LLC
|
|
2
|
+
#
|
|
3
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
4
|
+
# you may not use this file except in compliance with the License.
|
|
5
|
+
# You may obtain a copy of the License at
|
|
6
|
+
#
|
|
7
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
|
8
|
+
#
|
|
9
|
+
# Unless required by applicable law or agreed to in writing, software
|
|
10
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
11
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
12
|
+
# See the License for the specific language governing permissions and
|
|
13
|
+
# limitations under the License.
|
|
14
|
+
|
|
15
|
+
"""Helpers shared by the tests of `app/app_utils/claude.py` and `turn_options.py`."""
|
|
16
|
+
|
|
17
|
+
import pathlib
|
|
18
|
+
|
|
19
|
+
from app.app_utils import settings, turn_options, workspaces
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
def workspaces_root() -> pathlib.Path:
|
|
23
|
+
"""Where the agent's conversation workspaces go."""
|
|
24
|
+
return workspaces.WORKSPACES / settings.AGENT_NAME
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
def write_local_transcript(conversation_id: str) -> None:
|
|
28
|
+
"""What the CLI leaves on disk after a conversation's first turn."""
|
|
29
|
+
session = turn_options.session_id_for(conversation_id)
|
|
30
|
+
transcript = workspaces.transcripts_dir(workspaces_root() / session)
|
|
31
|
+
transcript.mkdir(parents=True, exist_ok=True)
|
|
32
|
+
(transcript / f"{session}.jsonl").write_text('{"type": "user"}\n')
|