google-agents-cli-extensions 1.14.2__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (107) hide show
  1. core/claude/README.md +35 -0
  2. core/claude/template/.agents/skills/agents-cli-claude/SKILL.md +106 -0
  3. core/claude/template/.agents/skills/agents-cli-claude/references/claude.md +386 -0
  4. core/claude/template/.agents/skills/agents-cli-claude/references/migrate-claude-agent.md +318 -0
  5. core/claude/template/.agents/skills/agents-cli-claude/references/migrate-from-aws.md +132 -0
  6. core/claude/template/.dockerignore +28 -0
  7. core/claude/template/.env +18 -0
  8. core/claude/template/.env.example +17 -0
  9. core/claude/template/.template/templateconfig.yaml +31 -0
  10. core/claude/template/AGENTS.md +33 -0
  11. core/claude/template/Dockerfile +55 -0
  12. core/claude/template/agent_config/README.md +9 -0
  13. core/claude/template/agents-cli-extension.yaml +49 -0
  14. core/claude/template/app/__init__.py +15 -0
  15. core/claude/template/app/agent.py +49 -0
  16. core/claude/template/app/app_utils/a2a.py +245 -0
  17. core/claude/template/app/app_utils/bq_analytics/__init__.py +124 -0
  18. core/claude/template/app/app_utils/bq_analytics/recorder.py +593 -0
  19. core/claude/template/app/app_utils/bq_analytics/redaction.py +209 -0
  20. core/claude/template/app/app_utils/bq_analytics/schema.py +394 -0
  21. core/claude/template/app/app_utils/bq_analytics/tables.py +104 -0
  22. core/claude/template/app/app_utils/bq_analytics/uploader.py +188 -0
  23. core/claude/template/app/app_utils/claude.py +297 -0
  24. core/claude/template/app/app_utils/completions.py +225 -0
  25. core/claude/template/app/app_utils/conversation.py +29 -0
  26. core/claude/template/app/app_utils/memory_bank.py +308 -0
  27. core/claude/template/app/app_utils/project_settings.py +91 -0
  28. core/claude/template/app/app_utils/reasoning_engine.py +237 -0
  29. core/claude/template/app/app_utils/remote_agents.py +349 -0
  30. core/claude/template/app/app_utils/remote_auth.py +159 -0
  31. core/claude/template/app/app_utils/replies.py +143 -0
  32. core/claude/template/app/app_utils/session_store.py +174 -0
  33. core/claude/template/app/app_utils/settings.py +24 -0
  34. core/claude/template/app/app_utils/telemetry/__init__.py +114 -0
  35. core/claude/template/app/app_utils/telemetry/call_counts.py +44 -0
  36. core/claude/template/app/app_utils/telemetry/cli_relay.py +266 -0
  37. core/claude/template/app/app_utils/telemetry/config.py +133 -0
  38. core/claude/template/app/app_utils/telemetry/logs.py +175 -0
  39. core/claude/template/app/app_utils/telemetry/metrics.py +270 -0
  40. core/claude/template/app/app_utils/telemetry/request_flush.py +104 -0
  41. core/claude/template/app/app_utils/telemetry/subagent_spans.py +80 -0
  42. core/claude/template/app/app_utils/telemetry/traces.py +166 -0
  43. core/claude/template/app/app_utils/turn_events.py +463 -0
  44. core/claude/template/app/app_utils/turn_options.py +219 -0
  45. core/claude/template/app/app_utils/workspaces.py +65 -0
  46. core/claude/template/app/fast_api_app.py +71 -0
  47. core/claude/template/app/feature_tools.py +27 -0
  48. core/claude/template/pyproject.toml +100 -0
  49. core/claude/template/scripts/eval_generate.py +340 -0
  50. core/claude/template/scripts/publish.py +66 -0
  51. core/claude/template/scripts/run.py +85 -0
  52. core/claude/template/scripts/unsupported.py +48 -0
  53. core/claude/template/tests/eval/datasets/basic-dataset.json +31 -0
  54. core/claude/template/tests/eval/eval_config.yaml +17 -0
  55. core/claude/template/tests/eval/response_quality.py +60 -0
  56. core/claude/template/tests/integration/test_server_e2e.py +168 -0
  57. core/claude/template/tests/load_test/.results/.placeholder +0 -0
  58. core/claude/template/tests/load_test/README.md +24 -0
  59. core/claude/template/tests/load_test/load_test.py +118 -0
  60. core/claude/template/tests/unit/conftest.py +63 -0
  61. core/claude/template/tests/unit/helpers.py +32 -0
  62. core/claude/template/tests/unit/test_a2a.py +432 -0
  63. core/claude/template/tests/unit/test_bq_analytics.py +928 -0
  64. core/claude/template/tests/unit/test_bq_redaction.py +89 -0
  65. core/claude/template/tests/unit/test_claude.py +825 -0
  66. core/claude/template/tests/unit/test_completions.py +369 -0
  67. core/claude/template/tests/unit/test_fast_api_app.py +59 -0
  68. core/claude/template/tests/unit/test_feature_tools.py +55 -0
  69. core/claude/template/tests/unit/test_memory_bank.py +625 -0
  70. core/claude/template/tests/unit/test_reasoning_engine.py +295 -0
  71. core/claude/template/tests/unit/test_remote_agents.py +517 -0
  72. core/claude/template/tests/unit/test_telemetry.py +991 -0
  73. core/claude/template/tests/unit/test_turn_events.py +309 -0
  74. core/claude/template/tests/unit/test_turn_options.py +394 -0
  75. core/claude/template/uv.lock +3029 -0
  76. core/langchain/README.md +54 -0
  77. core/langchain/template/.agents/skills/agents-cli-langchain/SKILL.md +84 -0
  78. core/langchain/template/.agents/skills/agents-cli-langchain/references/langchain.md +120 -0
  79. core/langchain/template/.agents/skills/agents-cli-langchain/references/samples.md +42 -0
  80. core/langchain/template/.template/templateconfig.yaml +32 -0
  81. core/langchain/template/AGENTS.md +35 -0
  82. core/langchain/template/agents-cli-extension.yaml +54 -0
  83. core/langchain/template/app/__init__.py +19 -0
  84. core/langchain/template/app/agent.py +61 -0
  85. core/langchain/template/app/app_utils/a2a.py +176 -0
  86. core/langchain/template/app/app_utils/content.py +37 -0
  87. core/langchain/template/app/app_utils/telemetry.py +154 -0
  88. core/langchain/template/app/fast_api_app.py +68 -0
  89. core/langchain/template/langgraph.json +7 -0
  90. core/langchain/template/pyproject.toml +102 -0
  91. core/langchain/template/scripts/_content.py +40 -0
  92. core/langchain/template/scripts/eval_generate.py +202 -0
  93. core/langchain/template/scripts/publish.py +59 -0
  94. core/langchain/template/scripts/run.py +59 -0
  95. core/langchain/template/scripts/unsupported.py +48 -0
  96. core/langchain/template/tests/eval/datasets/basic-dataset.json +16 -0
  97. core/langchain/template/tests/eval/eval_config.yaml +32 -0
  98. core/langchain/template/tests/eval/response_quality.py +60 -0
  99. core/langchain/template/tests/integration/test_server_e2e.py +173 -0
  100. core/langchain/template/tests/load_test/.results/.placeholder +0 -0
  101. core/langchain/template/tests/load_test/README.md +20 -0
  102. core/langchain/template/tests/load_test/load_test.py +82 -0
  103. core/langchain/template/uv.lock +3820 -0
  104. google_agents_cli_extensions-1.14.2.dist-info/METADATA +133 -0
  105. google_agents_cli_extensions-1.14.2.dist-info/RECORD +107 -0
  106. google_agents_cli_extensions-1.14.2.dist-info/WHEEL +4 -0
  107. google_agents_cli_extensions-1.14.2.dist-info/licenses/LICENSE +201 -0
@@ -0,0 +1,202 @@
1
+ # Copyright 2026 Google LLC
2
+ #
3
+ # Licensed under the Apache License, Version 2.0 (the "License");
4
+ # you may not use this file except in compliance with the License.
5
+ # You may obtain a copy of the License at
6
+ #
7
+ # http://www.apache.org/licenses/LICENSE-2.0
8
+ #
9
+ # Unless required by applicable law or agreed to in writing, software
10
+ # distributed under the License is distributed on an "AS IS" BASIS,
11
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
+ # See the License for the specific language governing permissions and
13
+ # limitations under the License.
14
+
15
+ """LangChain trace generator — drop-in for `agents-cli eval generate`.
16
+
17
+ Produces the same EvaluationDataset shape the built-in `agents-cli eval generate`
18
+ emits, so `agents-cli eval grade` consumes the output unchanged.
19
+ """
20
+
21
+ from __future__ import annotations
22
+
23
+ import argparse
24
+ import datetime
25
+ import json
26
+ import logging
27
+ import os
28
+ import sys
29
+ from collections.abc import Callable
30
+ from pathlib import Path
31
+
32
+ from _content import content_to_text
33
+
34
+
35
+ def _resolve_output_path(output: str) -> Path:
36
+ p = Path(output)
37
+ # A trailing separator signals a directory that does not exist yet, so
38
+ # is_dir() alone misses it. Same rule as the built-in's `resolve_output_path`
39
+ # (google/agents/cli/eval/_paths.py), including os.sep for `traces\` on Windows.
40
+ if p.is_dir() or output.endswith(("/", os.sep)):
41
+ p.mkdir(parents=True, exist_ok=True)
42
+ stamp = datetime.datetime.now().strftime("%Y%m%d_%H%M%S")
43
+ return p / f"eval_dataset_{stamp}.json"
44
+ p.parent.mkdir(parents=True, exist_ok=True)
45
+ return p
46
+
47
+
48
+ def _agent_name() -> str:
49
+ default_name = "agent"
50
+ manifest = Path("agents-cli-manifest.yaml")
51
+ if not manifest.exists():
52
+ logging.warning(
53
+ "%s not found; using %r as the agent name.", manifest, default_name
54
+ )
55
+ return default_name
56
+ try:
57
+ import yaml
58
+
59
+ data = yaml.safe_load(manifest.read_text(encoding="utf-8")) or {}
60
+ return data.get("name") or default_name
61
+ except Exception as e:
62
+ logging.warning(
63
+ "Could not read the agent name from %s (%s); using %r.",
64
+ manifest,
65
+ e,
66
+ default_name,
67
+ )
68
+ return default_name
69
+
70
+
71
+ def _user_prompt(turn: dict) -> str | None:
72
+ # Walk newest-first so the most recent user message in the turn wins.
73
+ for event in reversed(turn.get("events") or []):
74
+ if event.get("author") == "user":
75
+ for part in event.get("content", {}).get("parts") or []:
76
+ if "text" in part:
77
+ return part["text"]
78
+ return None
79
+
80
+
81
+ def _prompt_text(prompt: object) -> str | None:
82
+ """Text of a case's top-level ``prompt`` (a Content: role + parts)."""
83
+ if isinstance(prompt, str):
84
+ return prompt
85
+ if isinstance(prompt, dict):
86
+ for part in prompt.get("parts") or []:
87
+ if isinstance(part, dict) and part.get("text"):
88
+ return part["text"]
89
+ return None
90
+
91
+
92
+ def generate_traces(
93
+ dataset: dict, agent_name: str, infer: Callable[[str], str]
94
+ ) -> dict:
95
+ """Fill each eval case with the agent's replies. Returns the same shape.
96
+
97
+ Matches what `eval grade` (the Eval Service in Gemini platform) requires of a trace:
98
+ every turn carries `turn_index`/`turn_id`, and the case carries the final
99
+ reply under `responses`. Without those the service rejects the file with
100
+ `turns[0].turn_index: Required field is not set` or scores nothing because
101
+ the response candidate is empty.
102
+
103
+ Handles both input shapes: a case with a top-level `prompt`, and a case
104
+ whose `agent_data.turns` end in a user event.
105
+ """
106
+ for case in dataset.get("eval_cases") or []:
107
+ agent_data = case.setdefault("agent_data", {})
108
+ turns = agent_data.setdefault("turns", [])
109
+ reply: str | None = None
110
+
111
+ for turn in turns:
112
+ prompt = _user_prompt(turn)
113
+ if prompt is None:
114
+ continue
115
+ reply = infer(prompt)
116
+ turn.setdefault("events", []).append(
117
+ {
118
+ "author": agent_name,
119
+ "content": {"role": "model", "parts": [{"text": reply}]},
120
+ }
121
+ )
122
+
123
+ # A prompt-shaped case has no turns to extend: the prompt stays where it
124
+ # is and the reply becomes turn 0, which is what the built-in generate
125
+ # produces for the same input.
126
+ if not turns:
127
+ prompt = _prompt_text(case.get("prompt"))
128
+ if prompt is not None:
129
+ reply = infer(prompt)
130
+ turns.append(
131
+ {
132
+ "events": [
133
+ {
134
+ "author": agent_name,
135
+ "content": {
136
+ "role": "model",
137
+ "parts": [{"text": reply}],
138
+ },
139
+ }
140
+ ]
141
+ }
142
+ )
143
+
144
+ for index, turn in enumerate(turns):
145
+ turn.setdefault("turn_index", index)
146
+ turn.setdefault("turn_id", f"turn_{index}")
147
+
148
+ if reply is not None:
149
+ case.setdefault("responses", []).append(
150
+ {"response": {"role": "model", "parts": [{"text": reply}]}}
151
+ )
152
+ return dataset
153
+
154
+
155
+ def main() -> int:
156
+ parser = argparse.ArgumentParser(description=__doc__)
157
+ # Same defaults as the built-in, so bare `eval generate` works here too.
158
+ parser.add_argument("--dataset", default="tests/eval/datasets/basic-dataset.json")
159
+ parser.add_argument("--output", "-o", default="artifacts/traces/")
160
+ # An override replaces the built-in, so it has to accept the built-in's argv:
161
+ # `eval run` forwards these, and argparse would otherwise exit 2 on them.
162
+ parser.add_argument("--url", default=None)
163
+ parser.add_argument("--app-name", default=None, help="Ignored; no server here.")
164
+ parser.add_argument(
165
+ "--concurrency", default=None, help="Ignored; cases run in order."
166
+ )
167
+ parser.add_argument(
168
+ "--header", action="append", default=[], help="Ignored; no server."
169
+ )
170
+ args = parser.parse_args()
171
+
172
+ if args.url:
173
+ print(
174
+ "langchain: --url is not supported; this project's `eval generate` invokes "
175
+ "the graph in process. To query a deployed agent, bypass the override: "
176
+ "AGENTS_CLI_DISABLE_OVERRIDES=1 agents-cli eval generate --url ...",
177
+ file=sys.stderr,
178
+ )
179
+ return 2
180
+
181
+ from app.agent import root_agent
182
+
183
+ def infer(prompt: str) -> str:
184
+ result = root_agent.invoke({"messages": [{"role": "user", "content": prompt}]})
185
+ last = result["messages"][-1]
186
+ # getattr handles LangChain BaseMessage objects; fall back to dict access
187
+ # for plain dicts. Avoid calling .get() on a BaseMessage (no such method).
188
+ content = getattr(last, "content", None)
189
+ if content is None and isinstance(last, dict):
190
+ content = last.get("content", "")
191
+ return content_to_text(content)
192
+
193
+ raw = json.loads(Path(args.dataset).read_text(encoding="utf-8"))
194
+ out = generate_traces(raw, _agent_name(), infer)
195
+ out_path = _resolve_output_path(args.output)
196
+ out_path.write_text(json.dumps(out, indent=2), encoding="utf-8")
197
+ print(f"[langchain-generate] wrote {out_path}")
198
+ return 0
199
+
200
+
201
+ if __name__ == "__main__":
202
+ sys.exit(main())
@@ -0,0 +1,59 @@
1
+ # Copyright 2026 Google LLC
2
+ #
3
+ # Licensed under the Apache License, Version 2.0 (the "License");
4
+ # you may not use this file except in compliance with the License.
5
+ # You may obtain a copy of the License at
6
+ #
7
+ # http://www.apache.org/licenses/LICENSE-2.0
8
+ #
9
+ # Unless required by applicable law or agreed to in writing, software
10
+ # distributed under the License is distributed on an "AS IS" BASIS,
11
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
+ # See the License for the specific language governing permissions and
13
+ # limitations under the License.
14
+
15
+ """Register with Gemini, except where the contract needs ADK.
16
+
17
+ On Agent Runtime the built-in registers the agent for native `:streamQuery`,
18
+ which only an ADK app serves. Everywhere else it registers over A2A, which this
19
+ project does serve, so the built-in is fine and runs unchanged.
20
+ """
21
+
22
+ from __future__ import annotations
23
+
24
+ import os
25
+ import pathlib
26
+ import subprocess
27
+ import sys
28
+
29
+ import yaml
30
+
31
+
32
+ def _deployment_target() -> str:
33
+ manifest = pathlib.Path("agents-cli-manifest.yaml")
34
+ if not manifest.exists():
35
+ return ""
36
+ data = yaml.safe_load(manifest.read_text(encoding="utf-8")) or {}
37
+ return str(data.get("create_params", {}).get("deployment_target", ""))
38
+
39
+
40
+ def main() -> int:
41
+ if _deployment_target() == "agent_runtime":
42
+ print(
43
+ "Publishing to Gemini is not supported by the langchain extension on "
44
+ "Agent Runtime: registration there invokes the agent with "
45
+ "ADK's :streamQuery contract, which this project does not serve. Deploy "
46
+ "to cloud_run or gke and publish from there, where registration is A2A.",
47
+ file=sys.stderr,
48
+ )
49
+ return 2
50
+
51
+ env = {**os.environ, "AGENTS_CLI_DISABLE_OVERRIDES": "1"}
52
+ # The pre-rename name, which every agents-cli version accepts.
53
+ return subprocess.run(
54
+ ["agents-cli", "publish", "gemini-enterprise", *sys.argv[1:]], env=env
55
+ ).returncode
56
+
57
+
58
+ if __name__ == "__main__":
59
+ sys.exit(main())
@@ -0,0 +1,59 @@
1
+ # Copyright 2026 Google LLC
2
+ #
3
+ # Licensed under the Apache License, Version 2.0 (the "License");
4
+ # you may not use this file except in compliance with the License.
5
+ # You may obtain a copy of the License at
6
+ #
7
+ # http://www.apache.org/licenses/LICENSE-2.0
8
+ #
9
+ # Unless required by applicable law or agreed to in writing, software
10
+ # distributed under the License is distributed on an "AS IS" BASIS,
11
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
+ # See the License for the specific language governing permissions and
13
+ # limitations under the License.
14
+
15
+ """One-shot agent-graph runner — drop-in for `agents-cli run`.
16
+
17
+ Invokes the compiled graph in-process (no server, no ports). For remote
18
+ queries against a deployed service, bypass the override:
19
+
20
+ AGENTS_CLI_DISABLE_OVERRIDES=1 agents-cli run --url https://... --mode a2a "hi"
21
+ """
22
+
23
+ from __future__ import annotations
24
+
25
+ import argparse
26
+ import sys
27
+
28
+ from _content import content_to_text
29
+
30
+
31
+ def main() -> int:
32
+ parser = argparse.ArgumentParser(prog="agents-cli run", description=__doc__)
33
+ parser.add_argument("message", help="The prompt to send to the agent.")
34
+ # Declared so the two remote flags get the bypass hint rather than
35
+ # argparse's bare "unrecognized arguments"; neither means anything here.
36
+ parser.add_argument("--url", default=None, help="Not supported by this override.")
37
+ parser.add_argument("--mode", default=None, help="Not supported by this override.")
38
+ args = parser.parse_args()
39
+
40
+ if args.url or args.mode:
41
+ print(
42
+ "langchain: --url/--mode are not supported; this project's `run` invokes "
43
+ "the graph in process. To query a deployed agent, bypass the override: "
44
+ 'AGENTS_CLI_DISABLE_OVERRIDES=1 agents-cli run --url ... --mode a2a "hi"',
45
+ file=sys.stderr,
46
+ )
47
+ return 2
48
+
49
+ from app.agent import root_agent
50
+
51
+ result = root_agent.invoke(
52
+ {"messages": [{"role": "user", "content": args.message}]}
53
+ )
54
+ print(content_to_text(result["messages"][-1].content))
55
+ return 0
56
+
57
+
58
+ if __name__ == "__main__":
59
+ sys.exit(main())
@@ -0,0 +1,48 @@
1
+ # Copyright 2026 Google LLC
2
+ #
3
+ # Licensed under the Apache License, Version 2.0 (the "License");
4
+ # you may not use this file except in compliance with the License.
5
+ # You may obtain a copy of the License at
6
+ #
7
+ # http://www.apache.org/licenses/LICENSE-2.0
8
+ #
9
+ # Unless required by applicable law or agreed to in writing, software
10
+ # distributed under the License is distributed on an "AS IS" BASIS,
11
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
+ # See the License for the specific language governing permissions and
13
+ # limitations under the License.
14
+
15
+ """Refuse a built-in that cannot work without ADK.
16
+
17
+ `eval dataset synthesize` and `eval optimize` drive the agent through ADK, so
18
+ they die on a missing `google.adk` import or a missing `adk` binary. Refusing
19
+ with the reason beats a traceback. The manifest points each command here by name.
20
+ """
21
+
22
+ from __future__ import annotations
23
+
24
+ import sys
25
+
26
+ _REASONS = {
27
+ "eval dataset": (
28
+ "it runs the agent through ADK's user simulator, which needs google-adk "
29
+ "in this project's venv."
30
+ ),
31
+ "eval optimize": "it shells out to the `adk` binary, which this project does not use.",
32
+ }
33
+
34
+
35
+ def main() -> int:
36
+ command = sys.argv[1] if len(sys.argv) > 1 else "this command"
37
+ reason = _REASONS.get(
38
+ command, "it depends on ADK, which this project does not use."
39
+ )
40
+ print(
41
+ f"agents-cli {command} is not supported by the langchain extension: {reason}",
42
+ file=sys.stderr,
43
+ )
44
+ return 2
45
+
46
+
47
+ if __name__ == "__main__":
48
+ sys.exit(main())
@@ -0,0 +1,16 @@
1
+ {
2
+ "eval_cases": [
3
+ {
4
+ "name": "greeting",
5
+ "agent_data": {
6
+ "turns": [
7
+ {
8
+ "events": [
9
+ {"author": "user", "content": {"parts": [{"text": "hello"}]}}
10
+ ]
11
+ }
12
+ ]
13
+ }
14
+ }
15
+ ]
16
+ }
@@ -0,0 +1,32 @@
1
+ # Copyright 2026 Google LLC
2
+ #
3
+ # Licensed under the Apache License, Version 2.0 (the "License");
4
+ # you may not use this file except in compliance with the License.
5
+ # You may obtain a copy of the License at
6
+ #
7
+ # https://www.apache.org/licenses/LICENSE-2.0
8
+ #
9
+ # Unless required by applicable law or agreed to in writing, software
10
+ # distributed under the License is distributed on an "AS IS" BASIS,
11
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
+ # See the License for the specific language governing permissions and
13
+ # limitations under the License.
14
+
15
+ metrics_to_run:
16
+ - custom_response_quality
17
+
18
+ # Fail `agents-cli eval run` (exit 1) when a metric's mean score drops below
19
+ # its threshold. Set it once you know the bar this agent must meet.
20
+ # thresholds:
21
+ # custom_response_quality: 4 # the judge scores 1-5
22
+
23
+ custom_metrics:
24
+ # Default: local LLM-as-judge (see response_quality.py). Runs in-process via
25
+ # google-genai on Gemini platform (ADC) or Google AI Studio (GEMINI_API_KEY).
26
+ - name: custom_response_quality
27
+ custom_function_file: response_quality.py
28
+ - name: agent_turn_count
29
+ custom_function: |
30
+ def evaluate(instance):
31
+ turns = (instance.get("agent_data") or {}).get("turns", [])
32
+ return {'score': len(turns)}
@@ -0,0 +1,60 @@
1
+ # Copyright 2026 Google LLC
2
+ #
3
+ # Licensed under the Apache License, Version 2.0 (the "License");
4
+ # you may not use this file except in compliance with the License.
5
+ # You may obtain a copy of the License at
6
+ #
7
+ # https://www.apache.org/licenses/LICENSE-2.0
8
+ #
9
+ # Unless required by applicable law or agreed to in writing, software
10
+ # distributed under the License is distributed on an "AS IS" BASIS,
11
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
+ # See the License for the specific language governing permissions and
13
+ # limitations under the License.
14
+
15
+ """Local LLM-as-judge for `custom_response_quality` (see eval_config.yaml)."""
16
+
17
+ from google import genai
18
+ from google.genai import types
19
+ from pydantic import BaseModel
20
+
21
+
22
+ class _Verdict(BaseModel):
23
+ score: int # 1-5
24
+ explanation: str
25
+
26
+
27
+ def evaluate(instance):
28
+ reference = instance.get("reference")
29
+ rubric = (
30
+ "Grade the agent's final response on a 1-5 scale (1 poor, 5 excellent) for "
31
+ "accuracy, relevance, and clarity."
32
+ )
33
+ if reference:
34
+ rubric += (
35
+ " The response should agree with the expected answer below; penalize "
36
+ "factual disagreement with it."
37
+ )
38
+ prompt = (
39
+ f"You are an expert QA evaluator for an enterprise AI assistant. {rubric}\n"
40
+ f"User Prompt: {instance.get('prompt', '')}\n"
41
+ f"Final Response: {instance.get('response', '')}\n"
42
+ )
43
+ if reference:
44
+ prompt += f"Expected Answer (ground truth): {reference}\n"
45
+ prompt += f"Full Agent Trace: {instance.get('agent_data', '')}\n"
46
+
47
+ client = genai.Client() # AI Studio (GEMINI_API_KEY) or Gemini platform (ADC)
48
+ response = client.models.generate_content(
49
+ model="gemini-3.8-flash",
50
+ contents=prompt,
51
+ config=types.GenerateContentConfig(
52
+ temperature=0, # deterministic grading
53
+ response_mime_type="application/json",
54
+ response_schema=_Verdict, # guaranteed schema-valid JSON
55
+ ),
56
+ )
57
+ verdict = response.parsed
58
+ if verdict is None: # model returned nothing usable
59
+ return {"score": 0, "explanation": response.text or ""}
60
+ return {"score": max(1, min(5, verdict.score)), "explanation": verdict.explanation}
@@ -0,0 +1,173 @@
1
+ # Copyright 2026 Google LLC
2
+ #
3
+ # Licensed under the Apache License, Version 2.0 (the "License");
4
+ # you may not use this file except in compliance with the License.
5
+ # You may obtain a copy of the License at
6
+ #
7
+ # http://www.apache.org/licenses/LICENSE-2.0
8
+ #
9
+ # Unless required by applicable law or agreed to in writing, software
10
+ # distributed under the License is distributed on an "AS IS" BASIS,
11
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
+ # See the License for the specific language governing permissions and
13
+ # limitations under the License.
14
+
15
+ """End-to-end test against the server this project actually runs.
16
+
17
+ Replaces the deployment target's copy, which drives routes only an ADK server
18
+ has. This app serves A2A, at the paths the deploy entrypoint advertises.
19
+ """
20
+
21
+ import asyncio
22
+ import logging
23
+ import os
24
+ import subprocess
25
+ import sys
26
+ import threading
27
+ import time
28
+ import uuid
29
+ from collections.abc import Iterator
30
+ from typing import Any
31
+
32
+ import httpx
33
+ import pytest
34
+ import requests
35
+ from a2a.client import ClientConfig, create_client
36
+ from a2a.types import (
37
+ Message,
38
+ Part,
39
+ Role,
40
+ SendMessageRequest,
41
+ StreamResponse,
42
+ TaskState,
43
+ )
44
+ from requests.exceptions import RequestException
45
+
46
+ logging.basicConfig(level=logging.INFO)
47
+ logger = logging.getLogger(__name__)
48
+
49
+ BASE_URL = "http://127.0.0.1:8000"
50
+ A2A_RPC_URL = BASE_URL + "/a2a/app/"
51
+ AGENT_CARD_URL = A2A_RPC_URL + ".well-known/agent-card.json"
52
+
53
+
54
+ def log_output(pipe: Any, log_func: Any) -> None:
55
+ for line in iter(pipe.readline, ""):
56
+ log_func(line.strip())
57
+
58
+
59
+ def start_server() -> subprocess.Popen[str]:
60
+ command = [
61
+ sys.executable,
62
+ "-m",
63
+ "uvicorn",
64
+ "app.fast_api_app:app",
65
+ "--host",
66
+ "0.0.0.0",
67
+ "--port",
68
+ "8000",
69
+ ]
70
+ env = os.environ.copy()
71
+ env["INTEGRATION_TEST"] = "TRUE"
72
+ # The agent card advertises this, and the A2A client follows it.
73
+ env["APP_URL"] = BASE_URL
74
+ process = subprocess.Popen(
75
+ command,
76
+ stdout=subprocess.PIPE,
77
+ stderr=subprocess.PIPE,
78
+ text=True,
79
+ bufsize=1,
80
+ env=env,
81
+ )
82
+ threading.Thread(
83
+ target=log_output, args=(process.stdout, logger.info), daemon=True
84
+ ).start()
85
+ threading.Thread(
86
+ target=log_output, args=(process.stderr, logger.error), daemon=True
87
+ ).start()
88
+ return process
89
+
90
+
91
+ def wait_for_server(timeout: int = 90, interval: int = 1) -> bool:
92
+ """The card is only served once the lifespan has built the graph."""
93
+ start_time = time.time()
94
+ while time.time() - start_time < timeout:
95
+ try:
96
+ if requests.get(AGENT_CARD_URL, timeout=10).status_code == 200:
97
+ logger.info("Server is ready")
98
+ return True
99
+ except RequestException:
100
+ pass
101
+ time.sleep(interval)
102
+ logger.error(f"Server did not become ready within {timeout} seconds")
103
+ return False
104
+
105
+
106
+ @pytest.fixture(scope="session")
107
+ def server_fixture(request: Any) -> Iterator[subprocess.Popen[str]]:
108
+ logger.info("Starting server process")
109
+ server_process = start_server()
110
+ if not wait_for_server():
111
+ pytest.fail("Server failed to start")
112
+
113
+ def stop_server() -> None:
114
+ server_process.terminate()
115
+ server_process.wait()
116
+
117
+ request.addfinalizer(stop_server)
118
+ yield server_process
119
+
120
+
121
+ def test_a2a_chat_stream(server_fixture: subprocess.Popen[str]) -> None:
122
+ """The graph answers over the A2A JSON-RPC streaming protocol."""
123
+
124
+ async def _stream() -> list[StreamResponse]:
125
+ config = ClientConfig(
126
+ streaming=True, httpx_client=httpx.AsyncClient(timeout=60.0)
127
+ )
128
+ client = await create_client(A2A_RPC_URL.rstrip("/"), config)
129
+ message = Message(
130
+ message_id=f"msg-user-{uuid.uuid4()}",
131
+ role=Role.ROLE_USER,
132
+ parts=[Part(text="Hi!")],
133
+ )
134
+ return [
135
+ chunk
136
+ async for chunk in client.send_message(SendMessageRequest(message=message))
137
+ ]
138
+
139
+ responses = asyncio.run(_stream())
140
+ assert responses, "No responses received from stream"
141
+
142
+ def _is_completed(chunk: StreamResponse) -> bool:
143
+ if chunk.HasField("status_update"):
144
+ return chunk.status_update.status.state == TaskState.TASK_STATE_COMPLETED
145
+ if chunk.HasField("task"):
146
+ return chunk.task.status.state == TaskState.TASK_STATE_COMPLETED
147
+ return False
148
+
149
+ assert any(_is_completed(chunk) for chunk in responses), (
150
+ "No completed task received from stream"
151
+ )
152
+
153
+
154
+ def test_agent_card(server_fixture: subprocess.Popen[str]) -> None:
155
+ """Deploy and `agents-cli run --mode a2a` both read this card."""
156
+ response = requests.get(AGENT_CARD_URL, timeout=10)
157
+ assert response.status_code == 200, f"A2A endpoint returned {response.status_code}"
158
+
159
+ served_agent_card = response.json()
160
+ for field in (
161
+ "name",
162
+ "description",
163
+ "skills",
164
+ "capabilities",
165
+ "version",
166
+ "supportedInterfaces",
167
+ ):
168
+ assert field in served_agent_card, f"Missing field in agent card: {field}"
169
+
170
+
171
+ def test_health(server_fixture: subprocess.Popen[str]) -> None:
172
+ """Cloud Run and GKE probe this."""
173
+ assert requests.get(f"{BASE_URL}/health", timeout=10).status_code == 200
@@ -0,0 +1,20 @@
1
+ # Load testing
2
+
3
+ [Locust](http://locust.io) drives the A2A `message/send` route this project serves.
4
+
5
+ Local, against `uv run uvicorn app.fast_api_app:app --port 8000`:
6
+
7
+ ```bash
8
+ uv run --with locust locust -f tests/load_test/load_test.py -H http://127.0.0.1:8000 \
9
+ -u 10 -r 2 -t 30s --headless
10
+ ```
11
+
12
+ Against a deployed service, send an identity token and point at its URL:
13
+
14
+ ```bash
15
+ export _ID_TOKEN=$(gcloud auth print-identity-token -q)
16
+ uv run --with locust locust -f tests/load_test/load_test.py -H "$RUN_SERVICE_URL" \
17
+ -u 10 -r 2 -t 60s --headless --csv=.results/report
18
+ ```
19
+
20
+ `--csv` writes the latency and failure tables Locust prints at the end.