google-agents-cli-extensions 1.14.2__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- core/claude/README.md +35 -0
- core/claude/template/.agents/skills/agents-cli-claude/SKILL.md +106 -0
- core/claude/template/.agents/skills/agents-cli-claude/references/claude.md +386 -0
- core/claude/template/.agents/skills/agents-cli-claude/references/migrate-claude-agent.md +318 -0
- core/claude/template/.agents/skills/agents-cli-claude/references/migrate-from-aws.md +132 -0
- core/claude/template/.dockerignore +28 -0
- core/claude/template/.env +18 -0
- core/claude/template/.env.example +17 -0
- core/claude/template/.template/templateconfig.yaml +31 -0
- core/claude/template/AGENTS.md +33 -0
- core/claude/template/Dockerfile +55 -0
- core/claude/template/agent_config/README.md +9 -0
- core/claude/template/agents-cli-extension.yaml +49 -0
- core/claude/template/app/__init__.py +15 -0
- core/claude/template/app/agent.py +49 -0
- core/claude/template/app/app_utils/a2a.py +245 -0
- core/claude/template/app/app_utils/bq_analytics/__init__.py +124 -0
- core/claude/template/app/app_utils/bq_analytics/recorder.py +593 -0
- core/claude/template/app/app_utils/bq_analytics/redaction.py +209 -0
- core/claude/template/app/app_utils/bq_analytics/schema.py +394 -0
- core/claude/template/app/app_utils/bq_analytics/tables.py +104 -0
- core/claude/template/app/app_utils/bq_analytics/uploader.py +188 -0
- core/claude/template/app/app_utils/claude.py +297 -0
- core/claude/template/app/app_utils/completions.py +225 -0
- core/claude/template/app/app_utils/conversation.py +29 -0
- core/claude/template/app/app_utils/memory_bank.py +308 -0
- core/claude/template/app/app_utils/project_settings.py +91 -0
- core/claude/template/app/app_utils/reasoning_engine.py +237 -0
- core/claude/template/app/app_utils/remote_agents.py +349 -0
- core/claude/template/app/app_utils/remote_auth.py +159 -0
- core/claude/template/app/app_utils/replies.py +143 -0
- core/claude/template/app/app_utils/session_store.py +174 -0
- core/claude/template/app/app_utils/settings.py +24 -0
- core/claude/template/app/app_utils/telemetry/__init__.py +114 -0
- core/claude/template/app/app_utils/telemetry/call_counts.py +44 -0
- core/claude/template/app/app_utils/telemetry/cli_relay.py +266 -0
- core/claude/template/app/app_utils/telemetry/config.py +133 -0
- core/claude/template/app/app_utils/telemetry/logs.py +175 -0
- core/claude/template/app/app_utils/telemetry/metrics.py +270 -0
- core/claude/template/app/app_utils/telemetry/request_flush.py +104 -0
- core/claude/template/app/app_utils/telemetry/subagent_spans.py +80 -0
- core/claude/template/app/app_utils/telemetry/traces.py +166 -0
- core/claude/template/app/app_utils/turn_events.py +463 -0
- core/claude/template/app/app_utils/turn_options.py +219 -0
- core/claude/template/app/app_utils/workspaces.py +65 -0
- core/claude/template/app/fast_api_app.py +71 -0
- core/claude/template/app/feature_tools.py +27 -0
- core/claude/template/pyproject.toml +100 -0
- core/claude/template/scripts/eval_generate.py +340 -0
- core/claude/template/scripts/publish.py +66 -0
- core/claude/template/scripts/run.py +85 -0
- core/claude/template/scripts/unsupported.py +48 -0
- core/claude/template/tests/eval/datasets/basic-dataset.json +31 -0
- core/claude/template/tests/eval/eval_config.yaml +17 -0
- core/claude/template/tests/eval/response_quality.py +60 -0
- core/claude/template/tests/integration/test_server_e2e.py +168 -0
- core/claude/template/tests/load_test/.results/.placeholder +0 -0
- core/claude/template/tests/load_test/README.md +24 -0
- core/claude/template/tests/load_test/load_test.py +118 -0
- core/claude/template/tests/unit/conftest.py +63 -0
- core/claude/template/tests/unit/helpers.py +32 -0
- core/claude/template/tests/unit/test_a2a.py +432 -0
- core/claude/template/tests/unit/test_bq_analytics.py +928 -0
- core/claude/template/tests/unit/test_bq_redaction.py +89 -0
- core/claude/template/tests/unit/test_claude.py +825 -0
- core/claude/template/tests/unit/test_completions.py +369 -0
- core/claude/template/tests/unit/test_fast_api_app.py +59 -0
- core/claude/template/tests/unit/test_feature_tools.py +55 -0
- core/claude/template/tests/unit/test_memory_bank.py +625 -0
- core/claude/template/tests/unit/test_reasoning_engine.py +295 -0
- core/claude/template/tests/unit/test_remote_agents.py +517 -0
- core/claude/template/tests/unit/test_telemetry.py +991 -0
- core/claude/template/tests/unit/test_turn_events.py +309 -0
- core/claude/template/tests/unit/test_turn_options.py +394 -0
- core/claude/template/uv.lock +3029 -0
- core/langchain/README.md +54 -0
- core/langchain/template/.agents/skills/agents-cli-langchain/SKILL.md +84 -0
- core/langchain/template/.agents/skills/agents-cli-langchain/references/langchain.md +120 -0
- core/langchain/template/.agents/skills/agents-cli-langchain/references/samples.md +42 -0
- core/langchain/template/.template/templateconfig.yaml +32 -0
- core/langchain/template/AGENTS.md +35 -0
- core/langchain/template/agents-cli-extension.yaml +54 -0
- core/langchain/template/app/__init__.py +19 -0
- core/langchain/template/app/agent.py +61 -0
- core/langchain/template/app/app_utils/a2a.py +176 -0
- core/langchain/template/app/app_utils/content.py +37 -0
- core/langchain/template/app/app_utils/telemetry.py +154 -0
- core/langchain/template/app/fast_api_app.py +68 -0
- core/langchain/template/langgraph.json +7 -0
- core/langchain/template/pyproject.toml +102 -0
- core/langchain/template/scripts/_content.py +40 -0
- core/langchain/template/scripts/eval_generate.py +202 -0
- core/langchain/template/scripts/publish.py +59 -0
- core/langchain/template/scripts/run.py +59 -0
- core/langchain/template/scripts/unsupported.py +48 -0
- core/langchain/template/tests/eval/datasets/basic-dataset.json +16 -0
- core/langchain/template/tests/eval/eval_config.yaml +32 -0
- core/langchain/template/tests/eval/response_quality.py +60 -0
- core/langchain/template/tests/integration/test_server_e2e.py +173 -0
- core/langchain/template/tests/load_test/.results/.placeholder +0 -0
- core/langchain/template/tests/load_test/README.md +20 -0
- core/langchain/template/tests/load_test/load_test.py +82 -0
- core/langchain/template/uv.lock +3820 -0
- google_agents_cli_extensions-1.14.2.dist-info/METADATA +133 -0
- google_agents_cli_extensions-1.14.2.dist-info/RECORD +107 -0
- google_agents_cli_extensions-1.14.2.dist-info/WHEEL +4 -0
- google_agents_cli_extensions-1.14.2.dist-info/licenses/LICENSE +201 -0
|
@@ -0,0 +1,202 @@
|
|
|
1
|
+
# Copyright 2026 Google LLC
|
|
2
|
+
#
|
|
3
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
4
|
+
# you may not use this file except in compliance with the License.
|
|
5
|
+
# You may obtain a copy of the License at
|
|
6
|
+
#
|
|
7
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
|
8
|
+
#
|
|
9
|
+
# Unless required by applicable law or agreed to in writing, software
|
|
10
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
11
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
12
|
+
# See the License for the specific language governing permissions and
|
|
13
|
+
# limitations under the License.
|
|
14
|
+
|
|
15
|
+
"""LangChain trace generator — drop-in for `agents-cli eval generate`.
|
|
16
|
+
|
|
17
|
+
Produces the same EvaluationDataset shape the built-in `agents-cli eval generate`
|
|
18
|
+
emits, so `agents-cli eval grade` consumes the output unchanged.
|
|
19
|
+
"""
|
|
20
|
+
|
|
21
|
+
from __future__ import annotations
|
|
22
|
+
|
|
23
|
+
import argparse
|
|
24
|
+
import datetime
|
|
25
|
+
import json
|
|
26
|
+
import logging
|
|
27
|
+
import os
|
|
28
|
+
import sys
|
|
29
|
+
from collections.abc import Callable
|
|
30
|
+
from pathlib import Path
|
|
31
|
+
|
|
32
|
+
from _content import content_to_text
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
def _resolve_output_path(output: str) -> Path:
|
|
36
|
+
p = Path(output)
|
|
37
|
+
# A trailing separator signals a directory that does not exist yet, so
|
|
38
|
+
# is_dir() alone misses it. Same rule as the built-in's `resolve_output_path`
|
|
39
|
+
# (google/agents/cli/eval/_paths.py), including os.sep for `traces\` on Windows.
|
|
40
|
+
if p.is_dir() or output.endswith(("/", os.sep)):
|
|
41
|
+
p.mkdir(parents=True, exist_ok=True)
|
|
42
|
+
stamp = datetime.datetime.now().strftime("%Y%m%d_%H%M%S")
|
|
43
|
+
return p / f"eval_dataset_{stamp}.json"
|
|
44
|
+
p.parent.mkdir(parents=True, exist_ok=True)
|
|
45
|
+
return p
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
def _agent_name() -> str:
|
|
49
|
+
default_name = "agent"
|
|
50
|
+
manifest = Path("agents-cli-manifest.yaml")
|
|
51
|
+
if not manifest.exists():
|
|
52
|
+
logging.warning(
|
|
53
|
+
"%s not found; using %r as the agent name.", manifest, default_name
|
|
54
|
+
)
|
|
55
|
+
return default_name
|
|
56
|
+
try:
|
|
57
|
+
import yaml
|
|
58
|
+
|
|
59
|
+
data = yaml.safe_load(manifest.read_text(encoding="utf-8")) or {}
|
|
60
|
+
return data.get("name") or default_name
|
|
61
|
+
except Exception as e:
|
|
62
|
+
logging.warning(
|
|
63
|
+
"Could not read the agent name from %s (%s); using %r.",
|
|
64
|
+
manifest,
|
|
65
|
+
e,
|
|
66
|
+
default_name,
|
|
67
|
+
)
|
|
68
|
+
return default_name
|
|
69
|
+
|
|
70
|
+
|
|
71
|
+
def _user_prompt(turn: dict) -> str | None:
|
|
72
|
+
# Walk newest-first so the most recent user message in the turn wins.
|
|
73
|
+
for event in reversed(turn.get("events") or []):
|
|
74
|
+
if event.get("author") == "user":
|
|
75
|
+
for part in event.get("content", {}).get("parts") or []:
|
|
76
|
+
if "text" in part:
|
|
77
|
+
return part["text"]
|
|
78
|
+
return None
|
|
79
|
+
|
|
80
|
+
|
|
81
|
+
def _prompt_text(prompt: object) -> str | None:
|
|
82
|
+
"""Text of a case's top-level ``prompt`` (a Content: role + parts)."""
|
|
83
|
+
if isinstance(prompt, str):
|
|
84
|
+
return prompt
|
|
85
|
+
if isinstance(prompt, dict):
|
|
86
|
+
for part in prompt.get("parts") or []:
|
|
87
|
+
if isinstance(part, dict) and part.get("text"):
|
|
88
|
+
return part["text"]
|
|
89
|
+
return None
|
|
90
|
+
|
|
91
|
+
|
|
92
|
+
def generate_traces(
|
|
93
|
+
dataset: dict, agent_name: str, infer: Callable[[str], str]
|
|
94
|
+
) -> dict:
|
|
95
|
+
"""Fill each eval case with the agent's replies. Returns the same shape.
|
|
96
|
+
|
|
97
|
+
Matches what `eval grade` (the Eval Service in Gemini platform) requires of a trace:
|
|
98
|
+
every turn carries `turn_index`/`turn_id`, and the case carries the final
|
|
99
|
+
reply under `responses`. Without those the service rejects the file with
|
|
100
|
+
`turns[0].turn_index: Required field is not set` or scores nothing because
|
|
101
|
+
the response candidate is empty.
|
|
102
|
+
|
|
103
|
+
Handles both input shapes: a case with a top-level `prompt`, and a case
|
|
104
|
+
whose `agent_data.turns` end in a user event.
|
|
105
|
+
"""
|
|
106
|
+
for case in dataset.get("eval_cases") or []:
|
|
107
|
+
agent_data = case.setdefault("agent_data", {})
|
|
108
|
+
turns = agent_data.setdefault("turns", [])
|
|
109
|
+
reply: str | None = None
|
|
110
|
+
|
|
111
|
+
for turn in turns:
|
|
112
|
+
prompt = _user_prompt(turn)
|
|
113
|
+
if prompt is None:
|
|
114
|
+
continue
|
|
115
|
+
reply = infer(prompt)
|
|
116
|
+
turn.setdefault("events", []).append(
|
|
117
|
+
{
|
|
118
|
+
"author": agent_name,
|
|
119
|
+
"content": {"role": "model", "parts": [{"text": reply}]},
|
|
120
|
+
}
|
|
121
|
+
)
|
|
122
|
+
|
|
123
|
+
# A prompt-shaped case has no turns to extend: the prompt stays where it
|
|
124
|
+
# is and the reply becomes turn 0, which is what the built-in generate
|
|
125
|
+
# produces for the same input.
|
|
126
|
+
if not turns:
|
|
127
|
+
prompt = _prompt_text(case.get("prompt"))
|
|
128
|
+
if prompt is not None:
|
|
129
|
+
reply = infer(prompt)
|
|
130
|
+
turns.append(
|
|
131
|
+
{
|
|
132
|
+
"events": [
|
|
133
|
+
{
|
|
134
|
+
"author": agent_name,
|
|
135
|
+
"content": {
|
|
136
|
+
"role": "model",
|
|
137
|
+
"parts": [{"text": reply}],
|
|
138
|
+
},
|
|
139
|
+
}
|
|
140
|
+
]
|
|
141
|
+
}
|
|
142
|
+
)
|
|
143
|
+
|
|
144
|
+
for index, turn in enumerate(turns):
|
|
145
|
+
turn.setdefault("turn_index", index)
|
|
146
|
+
turn.setdefault("turn_id", f"turn_{index}")
|
|
147
|
+
|
|
148
|
+
if reply is not None:
|
|
149
|
+
case.setdefault("responses", []).append(
|
|
150
|
+
{"response": {"role": "model", "parts": [{"text": reply}]}}
|
|
151
|
+
)
|
|
152
|
+
return dataset
|
|
153
|
+
|
|
154
|
+
|
|
155
|
+
def main() -> int:
|
|
156
|
+
parser = argparse.ArgumentParser(description=__doc__)
|
|
157
|
+
# Same defaults as the built-in, so bare `eval generate` works here too.
|
|
158
|
+
parser.add_argument("--dataset", default="tests/eval/datasets/basic-dataset.json")
|
|
159
|
+
parser.add_argument("--output", "-o", default="artifacts/traces/")
|
|
160
|
+
# An override replaces the built-in, so it has to accept the built-in's argv:
|
|
161
|
+
# `eval run` forwards these, and argparse would otherwise exit 2 on them.
|
|
162
|
+
parser.add_argument("--url", default=None)
|
|
163
|
+
parser.add_argument("--app-name", default=None, help="Ignored; no server here.")
|
|
164
|
+
parser.add_argument(
|
|
165
|
+
"--concurrency", default=None, help="Ignored; cases run in order."
|
|
166
|
+
)
|
|
167
|
+
parser.add_argument(
|
|
168
|
+
"--header", action="append", default=[], help="Ignored; no server."
|
|
169
|
+
)
|
|
170
|
+
args = parser.parse_args()
|
|
171
|
+
|
|
172
|
+
if args.url:
|
|
173
|
+
print(
|
|
174
|
+
"langchain: --url is not supported; this project's `eval generate` invokes "
|
|
175
|
+
"the graph in process. To query a deployed agent, bypass the override: "
|
|
176
|
+
"AGENTS_CLI_DISABLE_OVERRIDES=1 agents-cli eval generate --url ...",
|
|
177
|
+
file=sys.stderr,
|
|
178
|
+
)
|
|
179
|
+
return 2
|
|
180
|
+
|
|
181
|
+
from app.agent import root_agent
|
|
182
|
+
|
|
183
|
+
def infer(prompt: str) -> str:
|
|
184
|
+
result = root_agent.invoke({"messages": [{"role": "user", "content": prompt}]})
|
|
185
|
+
last = result["messages"][-1]
|
|
186
|
+
# getattr handles LangChain BaseMessage objects; fall back to dict access
|
|
187
|
+
# for plain dicts. Avoid calling .get() on a BaseMessage (no such method).
|
|
188
|
+
content = getattr(last, "content", None)
|
|
189
|
+
if content is None and isinstance(last, dict):
|
|
190
|
+
content = last.get("content", "")
|
|
191
|
+
return content_to_text(content)
|
|
192
|
+
|
|
193
|
+
raw = json.loads(Path(args.dataset).read_text(encoding="utf-8"))
|
|
194
|
+
out = generate_traces(raw, _agent_name(), infer)
|
|
195
|
+
out_path = _resolve_output_path(args.output)
|
|
196
|
+
out_path.write_text(json.dumps(out, indent=2), encoding="utf-8")
|
|
197
|
+
print(f"[langchain-generate] wrote {out_path}")
|
|
198
|
+
return 0
|
|
199
|
+
|
|
200
|
+
|
|
201
|
+
if __name__ == "__main__":
|
|
202
|
+
sys.exit(main())
|
|
@@ -0,0 +1,59 @@
|
|
|
1
|
+
# Copyright 2026 Google LLC
|
|
2
|
+
#
|
|
3
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
4
|
+
# you may not use this file except in compliance with the License.
|
|
5
|
+
# You may obtain a copy of the License at
|
|
6
|
+
#
|
|
7
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
|
8
|
+
#
|
|
9
|
+
# Unless required by applicable law or agreed to in writing, software
|
|
10
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
11
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
12
|
+
# See the License for the specific language governing permissions and
|
|
13
|
+
# limitations under the License.
|
|
14
|
+
|
|
15
|
+
"""Register with Gemini, except where the contract needs ADK.
|
|
16
|
+
|
|
17
|
+
On Agent Runtime the built-in registers the agent for native `:streamQuery`,
|
|
18
|
+
which only an ADK app serves. Everywhere else it registers over A2A, which this
|
|
19
|
+
project does serve, so the built-in is fine and runs unchanged.
|
|
20
|
+
"""
|
|
21
|
+
|
|
22
|
+
from __future__ import annotations
|
|
23
|
+
|
|
24
|
+
import os
|
|
25
|
+
import pathlib
|
|
26
|
+
import subprocess
|
|
27
|
+
import sys
|
|
28
|
+
|
|
29
|
+
import yaml
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
def _deployment_target() -> str:
|
|
33
|
+
manifest = pathlib.Path("agents-cli-manifest.yaml")
|
|
34
|
+
if not manifest.exists():
|
|
35
|
+
return ""
|
|
36
|
+
data = yaml.safe_load(manifest.read_text(encoding="utf-8")) or {}
|
|
37
|
+
return str(data.get("create_params", {}).get("deployment_target", ""))
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
def main() -> int:
|
|
41
|
+
if _deployment_target() == "agent_runtime":
|
|
42
|
+
print(
|
|
43
|
+
"Publishing to Gemini is not supported by the langchain extension on "
|
|
44
|
+
"Agent Runtime: registration there invokes the agent with "
|
|
45
|
+
"ADK's :streamQuery contract, which this project does not serve. Deploy "
|
|
46
|
+
"to cloud_run or gke and publish from there, where registration is A2A.",
|
|
47
|
+
file=sys.stderr,
|
|
48
|
+
)
|
|
49
|
+
return 2
|
|
50
|
+
|
|
51
|
+
env = {**os.environ, "AGENTS_CLI_DISABLE_OVERRIDES": "1"}
|
|
52
|
+
# The pre-rename name, which every agents-cli version accepts.
|
|
53
|
+
return subprocess.run(
|
|
54
|
+
["agents-cli", "publish", "gemini-enterprise", *sys.argv[1:]], env=env
|
|
55
|
+
).returncode
|
|
56
|
+
|
|
57
|
+
|
|
58
|
+
if __name__ == "__main__":
|
|
59
|
+
sys.exit(main())
|
|
@@ -0,0 +1,59 @@
|
|
|
1
|
+
# Copyright 2026 Google LLC
|
|
2
|
+
#
|
|
3
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
4
|
+
# you may not use this file except in compliance with the License.
|
|
5
|
+
# You may obtain a copy of the License at
|
|
6
|
+
#
|
|
7
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
|
8
|
+
#
|
|
9
|
+
# Unless required by applicable law or agreed to in writing, software
|
|
10
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
11
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
12
|
+
# See the License for the specific language governing permissions and
|
|
13
|
+
# limitations under the License.
|
|
14
|
+
|
|
15
|
+
"""One-shot agent-graph runner — drop-in for `agents-cli run`.
|
|
16
|
+
|
|
17
|
+
Invokes the compiled graph in-process (no server, no ports). For remote
|
|
18
|
+
queries against a deployed service, bypass the override:
|
|
19
|
+
|
|
20
|
+
AGENTS_CLI_DISABLE_OVERRIDES=1 agents-cli run --url https://... --mode a2a "hi"
|
|
21
|
+
"""
|
|
22
|
+
|
|
23
|
+
from __future__ import annotations
|
|
24
|
+
|
|
25
|
+
import argparse
|
|
26
|
+
import sys
|
|
27
|
+
|
|
28
|
+
from _content import content_to_text
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
def main() -> int:
|
|
32
|
+
parser = argparse.ArgumentParser(prog="agents-cli run", description=__doc__)
|
|
33
|
+
parser.add_argument("message", help="The prompt to send to the agent.")
|
|
34
|
+
# Declared so the two remote flags get the bypass hint rather than
|
|
35
|
+
# argparse's bare "unrecognized arguments"; neither means anything here.
|
|
36
|
+
parser.add_argument("--url", default=None, help="Not supported by this override.")
|
|
37
|
+
parser.add_argument("--mode", default=None, help="Not supported by this override.")
|
|
38
|
+
args = parser.parse_args()
|
|
39
|
+
|
|
40
|
+
if args.url or args.mode:
|
|
41
|
+
print(
|
|
42
|
+
"langchain: --url/--mode are not supported; this project's `run` invokes "
|
|
43
|
+
"the graph in process. To query a deployed agent, bypass the override: "
|
|
44
|
+
'AGENTS_CLI_DISABLE_OVERRIDES=1 agents-cli run --url ... --mode a2a "hi"',
|
|
45
|
+
file=sys.stderr,
|
|
46
|
+
)
|
|
47
|
+
return 2
|
|
48
|
+
|
|
49
|
+
from app.agent import root_agent
|
|
50
|
+
|
|
51
|
+
result = root_agent.invoke(
|
|
52
|
+
{"messages": [{"role": "user", "content": args.message}]}
|
|
53
|
+
)
|
|
54
|
+
print(content_to_text(result["messages"][-1].content))
|
|
55
|
+
return 0
|
|
56
|
+
|
|
57
|
+
|
|
58
|
+
if __name__ == "__main__":
|
|
59
|
+
sys.exit(main())
|
|
@@ -0,0 +1,48 @@
|
|
|
1
|
+
# Copyright 2026 Google LLC
|
|
2
|
+
#
|
|
3
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
4
|
+
# you may not use this file except in compliance with the License.
|
|
5
|
+
# You may obtain a copy of the License at
|
|
6
|
+
#
|
|
7
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
|
8
|
+
#
|
|
9
|
+
# Unless required by applicable law or agreed to in writing, software
|
|
10
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
11
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
12
|
+
# See the License for the specific language governing permissions and
|
|
13
|
+
# limitations under the License.
|
|
14
|
+
|
|
15
|
+
"""Refuse a built-in that cannot work without ADK.
|
|
16
|
+
|
|
17
|
+
`eval dataset synthesize` and `eval optimize` drive the agent through ADK, so
|
|
18
|
+
they die on a missing `google.adk` import or a missing `adk` binary. Refusing
|
|
19
|
+
with the reason beats a traceback. The manifest points each command here by name.
|
|
20
|
+
"""
|
|
21
|
+
|
|
22
|
+
from __future__ import annotations
|
|
23
|
+
|
|
24
|
+
import sys
|
|
25
|
+
|
|
26
|
+
_REASONS = {
|
|
27
|
+
"eval dataset": (
|
|
28
|
+
"it runs the agent through ADK's user simulator, which needs google-adk "
|
|
29
|
+
"in this project's venv."
|
|
30
|
+
),
|
|
31
|
+
"eval optimize": "it shells out to the `adk` binary, which this project does not use.",
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
def main() -> int:
|
|
36
|
+
command = sys.argv[1] if len(sys.argv) > 1 else "this command"
|
|
37
|
+
reason = _REASONS.get(
|
|
38
|
+
command, "it depends on ADK, which this project does not use."
|
|
39
|
+
)
|
|
40
|
+
print(
|
|
41
|
+
f"agents-cli {command} is not supported by the langchain extension: {reason}",
|
|
42
|
+
file=sys.stderr,
|
|
43
|
+
)
|
|
44
|
+
return 2
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
if __name__ == "__main__":
|
|
48
|
+
sys.exit(main())
|
|
@@ -0,0 +1,32 @@
|
|
|
1
|
+
# Copyright 2026 Google LLC
|
|
2
|
+
#
|
|
3
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
4
|
+
# you may not use this file except in compliance with the License.
|
|
5
|
+
# You may obtain a copy of the License at
|
|
6
|
+
#
|
|
7
|
+
# https://www.apache.org/licenses/LICENSE-2.0
|
|
8
|
+
#
|
|
9
|
+
# Unless required by applicable law or agreed to in writing, software
|
|
10
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
11
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
12
|
+
# See the License for the specific language governing permissions and
|
|
13
|
+
# limitations under the License.
|
|
14
|
+
|
|
15
|
+
metrics_to_run:
|
|
16
|
+
- custom_response_quality
|
|
17
|
+
|
|
18
|
+
# Fail `agents-cli eval run` (exit 1) when a metric's mean score drops below
|
|
19
|
+
# its threshold. Set it once you know the bar this agent must meet.
|
|
20
|
+
# thresholds:
|
|
21
|
+
# custom_response_quality: 4 # the judge scores 1-5
|
|
22
|
+
|
|
23
|
+
custom_metrics:
|
|
24
|
+
# Default: local LLM-as-judge (see response_quality.py). Runs in-process via
|
|
25
|
+
# google-genai on Gemini platform (ADC) or Google AI Studio (GEMINI_API_KEY).
|
|
26
|
+
- name: custom_response_quality
|
|
27
|
+
custom_function_file: response_quality.py
|
|
28
|
+
- name: agent_turn_count
|
|
29
|
+
custom_function: |
|
|
30
|
+
def evaluate(instance):
|
|
31
|
+
turns = (instance.get("agent_data") or {}).get("turns", [])
|
|
32
|
+
return {'score': len(turns)}
|
|
@@ -0,0 +1,60 @@
|
|
|
1
|
+
# Copyright 2026 Google LLC
|
|
2
|
+
#
|
|
3
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
4
|
+
# you may not use this file except in compliance with the License.
|
|
5
|
+
# You may obtain a copy of the License at
|
|
6
|
+
#
|
|
7
|
+
# https://www.apache.org/licenses/LICENSE-2.0
|
|
8
|
+
#
|
|
9
|
+
# Unless required by applicable law or agreed to in writing, software
|
|
10
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
11
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
12
|
+
# See the License for the specific language governing permissions and
|
|
13
|
+
# limitations under the License.
|
|
14
|
+
|
|
15
|
+
"""Local LLM-as-judge for `custom_response_quality` (see eval_config.yaml)."""
|
|
16
|
+
|
|
17
|
+
from google import genai
|
|
18
|
+
from google.genai import types
|
|
19
|
+
from pydantic import BaseModel
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
class _Verdict(BaseModel):
|
|
23
|
+
score: int # 1-5
|
|
24
|
+
explanation: str
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
def evaluate(instance):
|
|
28
|
+
reference = instance.get("reference")
|
|
29
|
+
rubric = (
|
|
30
|
+
"Grade the agent's final response on a 1-5 scale (1 poor, 5 excellent) for "
|
|
31
|
+
"accuracy, relevance, and clarity."
|
|
32
|
+
)
|
|
33
|
+
if reference:
|
|
34
|
+
rubric += (
|
|
35
|
+
" The response should agree with the expected answer below; penalize "
|
|
36
|
+
"factual disagreement with it."
|
|
37
|
+
)
|
|
38
|
+
prompt = (
|
|
39
|
+
f"You are an expert QA evaluator for an enterprise AI assistant. {rubric}\n"
|
|
40
|
+
f"User Prompt: {instance.get('prompt', '')}\n"
|
|
41
|
+
f"Final Response: {instance.get('response', '')}\n"
|
|
42
|
+
)
|
|
43
|
+
if reference:
|
|
44
|
+
prompt += f"Expected Answer (ground truth): {reference}\n"
|
|
45
|
+
prompt += f"Full Agent Trace: {instance.get('agent_data', '')}\n"
|
|
46
|
+
|
|
47
|
+
client = genai.Client() # AI Studio (GEMINI_API_KEY) or Gemini platform (ADC)
|
|
48
|
+
response = client.models.generate_content(
|
|
49
|
+
model="gemini-3.8-flash",
|
|
50
|
+
contents=prompt,
|
|
51
|
+
config=types.GenerateContentConfig(
|
|
52
|
+
temperature=0, # deterministic grading
|
|
53
|
+
response_mime_type="application/json",
|
|
54
|
+
response_schema=_Verdict, # guaranteed schema-valid JSON
|
|
55
|
+
),
|
|
56
|
+
)
|
|
57
|
+
verdict = response.parsed
|
|
58
|
+
if verdict is None: # model returned nothing usable
|
|
59
|
+
return {"score": 0, "explanation": response.text or ""}
|
|
60
|
+
return {"score": max(1, min(5, verdict.score)), "explanation": verdict.explanation}
|
|
@@ -0,0 +1,173 @@
|
|
|
1
|
+
# Copyright 2026 Google LLC
|
|
2
|
+
#
|
|
3
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
4
|
+
# you may not use this file except in compliance with the License.
|
|
5
|
+
# You may obtain a copy of the License at
|
|
6
|
+
#
|
|
7
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
|
8
|
+
#
|
|
9
|
+
# Unless required by applicable law or agreed to in writing, software
|
|
10
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
11
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
12
|
+
# See the License for the specific language governing permissions and
|
|
13
|
+
# limitations under the License.
|
|
14
|
+
|
|
15
|
+
"""End-to-end test against the server this project actually runs.
|
|
16
|
+
|
|
17
|
+
Replaces the deployment target's copy, which drives routes only an ADK server
|
|
18
|
+
has. This app serves A2A, at the paths the deploy entrypoint advertises.
|
|
19
|
+
"""
|
|
20
|
+
|
|
21
|
+
import asyncio
|
|
22
|
+
import logging
|
|
23
|
+
import os
|
|
24
|
+
import subprocess
|
|
25
|
+
import sys
|
|
26
|
+
import threading
|
|
27
|
+
import time
|
|
28
|
+
import uuid
|
|
29
|
+
from collections.abc import Iterator
|
|
30
|
+
from typing import Any
|
|
31
|
+
|
|
32
|
+
import httpx
|
|
33
|
+
import pytest
|
|
34
|
+
import requests
|
|
35
|
+
from a2a.client import ClientConfig, create_client
|
|
36
|
+
from a2a.types import (
|
|
37
|
+
Message,
|
|
38
|
+
Part,
|
|
39
|
+
Role,
|
|
40
|
+
SendMessageRequest,
|
|
41
|
+
StreamResponse,
|
|
42
|
+
TaskState,
|
|
43
|
+
)
|
|
44
|
+
from requests.exceptions import RequestException
|
|
45
|
+
|
|
46
|
+
logging.basicConfig(level=logging.INFO)
|
|
47
|
+
logger = logging.getLogger(__name__)
|
|
48
|
+
|
|
49
|
+
BASE_URL = "http://127.0.0.1:8000"
|
|
50
|
+
A2A_RPC_URL = BASE_URL + "/a2a/app/"
|
|
51
|
+
AGENT_CARD_URL = A2A_RPC_URL + ".well-known/agent-card.json"
|
|
52
|
+
|
|
53
|
+
|
|
54
|
+
def log_output(pipe: Any, log_func: Any) -> None:
|
|
55
|
+
for line in iter(pipe.readline, ""):
|
|
56
|
+
log_func(line.strip())
|
|
57
|
+
|
|
58
|
+
|
|
59
|
+
def start_server() -> subprocess.Popen[str]:
|
|
60
|
+
command = [
|
|
61
|
+
sys.executable,
|
|
62
|
+
"-m",
|
|
63
|
+
"uvicorn",
|
|
64
|
+
"app.fast_api_app:app",
|
|
65
|
+
"--host",
|
|
66
|
+
"0.0.0.0",
|
|
67
|
+
"--port",
|
|
68
|
+
"8000",
|
|
69
|
+
]
|
|
70
|
+
env = os.environ.copy()
|
|
71
|
+
env["INTEGRATION_TEST"] = "TRUE"
|
|
72
|
+
# The agent card advertises this, and the A2A client follows it.
|
|
73
|
+
env["APP_URL"] = BASE_URL
|
|
74
|
+
process = subprocess.Popen(
|
|
75
|
+
command,
|
|
76
|
+
stdout=subprocess.PIPE,
|
|
77
|
+
stderr=subprocess.PIPE,
|
|
78
|
+
text=True,
|
|
79
|
+
bufsize=1,
|
|
80
|
+
env=env,
|
|
81
|
+
)
|
|
82
|
+
threading.Thread(
|
|
83
|
+
target=log_output, args=(process.stdout, logger.info), daemon=True
|
|
84
|
+
).start()
|
|
85
|
+
threading.Thread(
|
|
86
|
+
target=log_output, args=(process.stderr, logger.error), daemon=True
|
|
87
|
+
).start()
|
|
88
|
+
return process
|
|
89
|
+
|
|
90
|
+
|
|
91
|
+
def wait_for_server(timeout: int = 90, interval: int = 1) -> bool:
|
|
92
|
+
"""The card is only served once the lifespan has built the graph."""
|
|
93
|
+
start_time = time.time()
|
|
94
|
+
while time.time() - start_time < timeout:
|
|
95
|
+
try:
|
|
96
|
+
if requests.get(AGENT_CARD_URL, timeout=10).status_code == 200:
|
|
97
|
+
logger.info("Server is ready")
|
|
98
|
+
return True
|
|
99
|
+
except RequestException:
|
|
100
|
+
pass
|
|
101
|
+
time.sleep(interval)
|
|
102
|
+
logger.error(f"Server did not become ready within {timeout} seconds")
|
|
103
|
+
return False
|
|
104
|
+
|
|
105
|
+
|
|
106
|
+
@pytest.fixture(scope="session")
|
|
107
|
+
def server_fixture(request: Any) -> Iterator[subprocess.Popen[str]]:
|
|
108
|
+
logger.info("Starting server process")
|
|
109
|
+
server_process = start_server()
|
|
110
|
+
if not wait_for_server():
|
|
111
|
+
pytest.fail("Server failed to start")
|
|
112
|
+
|
|
113
|
+
def stop_server() -> None:
|
|
114
|
+
server_process.terminate()
|
|
115
|
+
server_process.wait()
|
|
116
|
+
|
|
117
|
+
request.addfinalizer(stop_server)
|
|
118
|
+
yield server_process
|
|
119
|
+
|
|
120
|
+
|
|
121
|
+
def test_a2a_chat_stream(server_fixture: subprocess.Popen[str]) -> None:
|
|
122
|
+
"""The graph answers over the A2A JSON-RPC streaming protocol."""
|
|
123
|
+
|
|
124
|
+
async def _stream() -> list[StreamResponse]:
|
|
125
|
+
config = ClientConfig(
|
|
126
|
+
streaming=True, httpx_client=httpx.AsyncClient(timeout=60.0)
|
|
127
|
+
)
|
|
128
|
+
client = await create_client(A2A_RPC_URL.rstrip("/"), config)
|
|
129
|
+
message = Message(
|
|
130
|
+
message_id=f"msg-user-{uuid.uuid4()}",
|
|
131
|
+
role=Role.ROLE_USER,
|
|
132
|
+
parts=[Part(text="Hi!")],
|
|
133
|
+
)
|
|
134
|
+
return [
|
|
135
|
+
chunk
|
|
136
|
+
async for chunk in client.send_message(SendMessageRequest(message=message))
|
|
137
|
+
]
|
|
138
|
+
|
|
139
|
+
responses = asyncio.run(_stream())
|
|
140
|
+
assert responses, "No responses received from stream"
|
|
141
|
+
|
|
142
|
+
def _is_completed(chunk: StreamResponse) -> bool:
|
|
143
|
+
if chunk.HasField("status_update"):
|
|
144
|
+
return chunk.status_update.status.state == TaskState.TASK_STATE_COMPLETED
|
|
145
|
+
if chunk.HasField("task"):
|
|
146
|
+
return chunk.task.status.state == TaskState.TASK_STATE_COMPLETED
|
|
147
|
+
return False
|
|
148
|
+
|
|
149
|
+
assert any(_is_completed(chunk) for chunk in responses), (
|
|
150
|
+
"No completed task received from stream"
|
|
151
|
+
)
|
|
152
|
+
|
|
153
|
+
|
|
154
|
+
def test_agent_card(server_fixture: subprocess.Popen[str]) -> None:
|
|
155
|
+
"""Deploy and `agents-cli run --mode a2a` both read this card."""
|
|
156
|
+
response = requests.get(AGENT_CARD_URL, timeout=10)
|
|
157
|
+
assert response.status_code == 200, f"A2A endpoint returned {response.status_code}"
|
|
158
|
+
|
|
159
|
+
served_agent_card = response.json()
|
|
160
|
+
for field in (
|
|
161
|
+
"name",
|
|
162
|
+
"description",
|
|
163
|
+
"skills",
|
|
164
|
+
"capabilities",
|
|
165
|
+
"version",
|
|
166
|
+
"supportedInterfaces",
|
|
167
|
+
):
|
|
168
|
+
assert field in served_agent_card, f"Missing field in agent card: {field}"
|
|
169
|
+
|
|
170
|
+
|
|
171
|
+
def test_health(server_fixture: subprocess.Popen[str]) -> None:
|
|
172
|
+
"""Cloud Run and GKE probe this."""
|
|
173
|
+
assert requests.get(f"{BASE_URL}/health", timeout=10).status_code == 200
|
|
File without changes
|
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
# Load testing
|
|
2
|
+
|
|
3
|
+
[Locust](http://locust.io) drives the A2A `message/send` route this project serves.
|
|
4
|
+
|
|
5
|
+
Local, against `uv run uvicorn app.fast_api_app:app --port 8000`:
|
|
6
|
+
|
|
7
|
+
```bash
|
|
8
|
+
uv run --with locust locust -f tests/load_test/load_test.py -H http://127.0.0.1:8000 \
|
|
9
|
+
-u 10 -r 2 -t 30s --headless
|
|
10
|
+
```
|
|
11
|
+
|
|
12
|
+
Against a deployed service, send an identity token and point at its URL:
|
|
13
|
+
|
|
14
|
+
```bash
|
|
15
|
+
export _ID_TOKEN=$(gcloud auth print-identity-token -q)
|
|
16
|
+
uv run --with locust locust -f tests/load_test/load_test.py -H "$RUN_SERVICE_URL" \
|
|
17
|
+
-u 10 -r 2 -t 60s --headless --csv=.results/report
|
|
18
|
+
```
|
|
19
|
+
|
|
20
|
+
`--csv` writes the latency and failure tables Locust prints at the end.
|