google-agents-cli-extensions 1.14.2__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- core/claude/README.md +35 -0
- core/claude/template/.agents/skills/agents-cli-claude/SKILL.md +106 -0
- core/claude/template/.agents/skills/agents-cli-claude/references/claude.md +386 -0
- core/claude/template/.agents/skills/agents-cli-claude/references/migrate-claude-agent.md +318 -0
- core/claude/template/.agents/skills/agents-cli-claude/references/migrate-from-aws.md +132 -0
- core/claude/template/.dockerignore +28 -0
- core/claude/template/.env +18 -0
- core/claude/template/.env.example +17 -0
- core/claude/template/.template/templateconfig.yaml +31 -0
- core/claude/template/AGENTS.md +33 -0
- core/claude/template/Dockerfile +55 -0
- core/claude/template/agent_config/README.md +9 -0
- core/claude/template/agents-cli-extension.yaml +49 -0
- core/claude/template/app/__init__.py +15 -0
- core/claude/template/app/agent.py +49 -0
- core/claude/template/app/app_utils/a2a.py +245 -0
- core/claude/template/app/app_utils/bq_analytics/__init__.py +124 -0
- core/claude/template/app/app_utils/bq_analytics/recorder.py +593 -0
- core/claude/template/app/app_utils/bq_analytics/redaction.py +209 -0
- core/claude/template/app/app_utils/bq_analytics/schema.py +394 -0
- core/claude/template/app/app_utils/bq_analytics/tables.py +104 -0
- core/claude/template/app/app_utils/bq_analytics/uploader.py +188 -0
- core/claude/template/app/app_utils/claude.py +297 -0
- core/claude/template/app/app_utils/completions.py +225 -0
- core/claude/template/app/app_utils/conversation.py +29 -0
- core/claude/template/app/app_utils/memory_bank.py +308 -0
- core/claude/template/app/app_utils/project_settings.py +91 -0
- core/claude/template/app/app_utils/reasoning_engine.py +237 -0
- core/claude/template/app/app_utils/remote_agents.py +349 -0
- core/claude/template/app/app_utils/remote_auth.py +159 -0
- core/claude/template/app/app_utils/replies.py +143 -0
- core/claude/template/app/app_utils/session_store.py +174 -0
- core/claude/template/app/app_utils/settings.py +24 -0
- core/claude/template/app/app_utils/telemetry/__init__.py +114 -0
- core/claude/template/app/app_utils/telemetry/call_counts.py +44 -0
- core/claude/template/app/app_utils/telemetry/cli_relay.py +266 -0
- core/claude/template/app/app_utils/telemetry/config.py +133 -0
- core/claude/template/app/app_utils/telemetry/logs.py +175 -0
- core/claude/template/app/app_utils/telemetry/metrics.py +270 -0
- core/claude/template/app/app_utils/telemetry/request_flush.py +104 -0
- core/claude/template/app/app_utils/telemetry/subagent_spans.py +80 -0
- core/claude/template/app/app_utils/telemetry/traces.py +166 -0
- core/claude/template/app/app_utils/turn_events.py +463 -0
- core/claude/template/app/app_utils/turn_options.py +219 -0
- core/claude/template/app/app_utils/workspaces.py +65 -0
- core/claude/template/app/fast_api_app.py +71 -0
- core/claude/template/app/feature_tools.py +27 -0
- core/claude/template/pyproject.toml +100 -0
- core/claude/template/scripts/eval_generate.py +340 -0
- core/claude/template/scripts/publish.py +66 -0
- core/claude/template/scripts/run.py +85 -0
- core/claude/template/scripts/unsupported.py +48 -0
- core/claude/template/tests/eval/datasets/basic-dataset.json +31 -0
- core/claude/template/tests/eval/eval_config.yaml +17 -0
- core/claude/template/tests/eval/response_quality.py +60 -0
- core/claude/template/tests/integration/test_server_e2e.py +168 -0
- core/claude/template/tests/load_test/.results/.placeholder +0 -0
- core/claude/template/tests/load_test/README.md +24 -0
- core/claude/template/tests/load_test/load_test.py +118 -0
- core/claude/template/tests/unit/conftest.py +63 -0
- core/claude/template/tests/unit/helpers.py +32 -0
- core/claude/template/tests/unit/test_a2a.py +432 -0
- core/claude/template/tests/unit/test_bq_analytics.py +928 -0
- core/claude/template/tests/unit/test_bq_redaction.py +89 -0
- core/claude/template/tests/unit/test_claude.py +825 -0
- core/claude/template/tests/unit/test_completions.py +369 -0
- core/claude/template/tests/unit/test_fast_api_app.py +59 -0
- core/claude/template/tests/unit/test_feature_tools.py +55 -0
- core/claude/template/tests/unit/test_memory_bank.py +625 -0
- core/claude/template/tests/unit/test_reasoning_engine.py +295 -0
- core/claude/template/tests/unit/test_remote_agents.py +517 -0
- core/claude/template/tests/unit/test_telemetry.py +991 -0
- core/claude/template/tests/unit/test_turn_events.py +309 -0
- core/claude/template/tests/unit/test_turn_options.py +394 -0
- core/claude/template/uv.lock +3029 -0
- core/langchain/README.md +54 -0
- core/langchain/template/.agents/skills/agents-cli-langchain/SKILL.md +84 -0
- core/langchain/template/.agents/skills/agents-cli-langchain/references/langchain.md +120 -0
- core/langchain/template/.agents/skills/agents-cli-langchain/references/samples.md +42 -0
- core/langchain/template/.template/templateconfig.yaml +32 -0
- core/langchain/template/AGENTS.md +35 -0
- core/langchain/template/agents-cli-extension.yaml +54 -0
- core/langchain/template/app/__init__.py +19 -0
- core/langchain/template/app/agent.py +61 -0
- core/langchain/template/app/app_utils/a2a.py +176 -0
- core/langchain/template/app/app_utils/content.py +37 -0
- core/langchain/template/app/app_utils/telemetry.py +154 -0
- core/langchain/template/app/fast_api_app.py +68 -0
- core/langchain/template/langgraph.json +7 -0
- core/langchain/template/pyproject.toml +102 -0
- core/langchain/template/scripts/_content.py +40 -0
- core/langchain/template/scripts/eval_generate.py +202 -0
- core/langchain/template/scripts/publish.py +59 -0
- core/langchain/template/scripts/run.py +59 -0
- core/langchain/template/scripts/unsupported.py +48 -0
- core/langchain/template/tests/eval/datasets/basic-dataset.json +16 -0
- core/langchain/template/tests/eval/eval_config.yaml +32 -0
- core/langchain/template/tests/eval/response_quality.py +60 -0
- core/langchain/template/tests/integration/test_server_e2e.py +173 -0
- core/langchain/template/tests/load_test/.results/.placeholder +0 -0
- core/langchain/template/tests/load_test/README.md +20 -0
- core/langchain/template/tests/load_test/load_test.py +82 -0
- core/langchain/template/uv.lock +3820 -0
- google_agents_cli_extensions-1.14.2.dist-info/METADATA +133 -0
- google_agents_cli_extensions-1.14.2.dist-info/RECORD +107 -0
- google_agents_cli_extensions-1.14.2.dist-info/WHEEL +4 -0
- google_agents_cli_extensions-1.14.2.dist-info/licenses/LICENSE +201 -0
|
@@ -0,0 +1,928 @@
|
|
|
1
|
+
# Copyright 2026 Google LLC
|
|
2
|
+
#
|
|
3
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
4
|
+
# you may not use this file except in compliance with the License.
|
|
5
|
+
# You may obtain a copy of the License at
|
|
6
|
+
#
|
|
7
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
|
8
|
+
#
|
|
9
|
+
# Unless required by applicable law or agreed to in writing, software
|
|
10
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
11
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
12
|
+
# See the License for the specific language governing permissions and
|
|
13
|
+
# limitations under the License.
|
|
14
|
+
|
|
15
|
+
"""BigQuery Agent Analytics rows, with BigQuery faked at the client."""
|
|
16
|
+
|
|
17
|
+
from __future__ import annotations
|
|
18
|
+
|
|
19
|
+
import json
|
|
20
|
+
import threading
|
|
21
|
+
from unittest.mock import ANY, MagicMock, patch
|
|
22
|
+
|
|
23
|
+
import pytest
|
|
24
|
+
from claude_agent_sdk import (
|
|
25
|
+
AssistantMessage,
|
|
26
|
+
ClaudeAgentOptions,
|
|
27
|
+
ResultMessage,
|
|
28
|
+
StreamEvent,
|
|
29
|
+
SystemMessage,
|
|
30
|
+
TextBlock,
|
|
31
|
+
ToolResultBlock,
|
|
32
|
+
ToolUseBlock,
|
|
33
|
+
UserMessage,
|
|
34
|
+
create_sdk_mcp_server,
|
|
35
|
+
)
|
|
36
|
+
from google.api_core.exceptions import NotFound
|
|
37
|
+
from google.cloud.bigquery.table import TableListItem
|
|
38
|
+
from opentelemetry.sdk.trace import TracerProvider
|
|
39
|
+
|
|
40
|
+
from app.app_utils import bq_analytics, settings
|
|
41
|
+
from app.app_utils.bq_analytics import schema as agent_events
|
|
42
|
+
from app.app_utils.bq_analytics import tables, uploader
|
|
43
|
+
from app.app_utils.bq_analytics.recorder import MAX_CONTENT_LENGTH, AgentEventsRecorder
|
|
44
|
+
from app.app_utils.remote_agents import remote_agent
|
|
45
|
+
from app.app_utils.telemetry.config import agent_name
|
|
46
|
+
from app.app_utils.turn_events import TurnParser
|
|
47
|
+
|
|
48
|
+
# The rows ADK's plugin writes for a turn with one tool call.
|
|
49
|
+
ADK_TOOL_TURN = [
|
|
50
|
+
"USER_MESSAGE_RECEIVED",
|
|
51
|
+
"INVOCATION_STARTING",
|
|
52
|
+
"AGENT_STARTING",
|
|
53
|
+
"LLM_REQUEST",
|
|
54
|
+
"LLM_RESPONSE",
|
|
55
|
+
"TOOL_STARTING",
|
|
56
|
+
"TOOL_COMPLETED",
|
|
57
|
+
"LLM_REQUEST",
|
|
58
|
+
"LLM_RESPONSE",
|
|
59
|
+
"AGENT_RESPONSE",
|
|
60
|
+
"AGENT_COMPLETED",
|
|
61
|
+
"INVOCATION_COMPLETED",
|
|
62
|
+
]
|
|
63
|
+
|
|
64
|
+
|
|
65
|
+
class FakeWriter:
|
|
66
|
+
def __init__(self) -> None:
|
|
67
|
+
self.rows: list[dict] = []
|
|
68
|
+
|
|
69
|
+
def put(self, row: dict) -> None:
|
|
70
|
+
self.rows.append(row)
|
|
71
|
+
|
|
72
|
+
|
|
73
|
+
def _event(kind: str, **fields) -> StreamEvent:
|
|
74
|
+
return StreamEvent(uuid="u", session_id="s", event={"type": kind, **fields})
|
|
75
|
+
|
|
76
|
+
|
|
77
|
+
def _model_call(message_id: str, blocks: list, output_tokens: int, stop: str) -> list:
|
|
78
|
+
"""A streamed model call as the CLI sends it."""
|
|
79
|
+
return [
|
|
80
|
+
_event("message_start", message={"usage": {"input_tokens": 100}}),
|
|
81
|
+
_event("content_block_delta", delta={"type": "text_delta", "text": "x"}),
|
|
82
|
+
AssistantMessage(
|
|
83
|
+
content=blocks,
|
|
84
|
+
model="claude-haiku-4-5",
|
|
85
|
+
message_id=message_id,
|
|
86
|
+
usage={"input_tokens": 100, "output_tokens": 1},
|
|
87
|
+
),
|
|
88
|
+
_event(
|
|
89
|
+
"message_delta",
|
|
90
|
+
usage={"output_tokens": output_tokens, "cache_read_input_tokens": 40},
|
|
91
|
+
delta={"stop_reason": stop},
|
|
92
|
+
),
|
|
93
|
+
_event("message_stop"),
|
|
94
|
+
]
|
|
95
|
+
|
|
96
|
+
|
|
97
|
+
def _result(**overrides) -> ResultMessage:
|
|
98
|
+
fields = {
|
|
99
|
+
"subtype": "success",
|
|
100
|
+
"duration_ms": 1,
|
|
101
|
+
"duration_api_ms": 1,
|
|
102
|
+
"is_error": False,
|
|
103
|
+
"num_turns": 2,
|
|
104
|
+
"session_id": "s",
|
|
105
|
+
"result": "It's foggy.",
|
|
106
|
+
"total_cost_usd": 0.01,
|
|
107
|
+
}
|
|
108
|
+
return ResultMessage(**{**fields, **overrides})
|
|
109
|
+
|
|
110
|
+
|
|
111
|
+
def _play(
|
|
112
|
+
parser: TurnParser, recorder, messages: list, error: BaseException | None = None
|
|
113
|
+
) -> None:
|
|
114
|
+
"""Feed `messages` through `parser` to `recorder`, as stream_turn does."""
|
|
115
|
+
for message in messages:
|
|
116
|
+
for event in parser.feed(message):
|
|
117
|
+
recorder.handle(event)
|
|
118
|
+
for event in parser.finish(error):
|
|
119
|
+
recorder.handle(event)
|
|
120
|
+
|
|
121
|
+
|
|
122
|
+
def _record(messages: list, **kwargs) -> list[dict]:
|
|
123
|
+
out = FakeWriter()
|
|
124
|
+
parser = TurnParser("weather_agent")
|
|
125
|
+
recorder = AgentEventsRecorder(
|
|
126
|
+
out,
|
|
127
|
+
prompt="Weather in SF?",
|
|
128
|
+
agent_name=parser.agent_name,
|
|
129
|
+
session_id="conv-1",
|
|
130
|
+
user_id="A2A_USER_conv-1",
|
|
131
|
+
model="claude-haiku-4-5",
|
|
132
|
+
system_prompt="You report the weather.",
|
|
133
|
+
instruction="You report the weather.",
|
|
134
|
+
**kwargs,
|
|
135
|
+
)
|
|
136
|
+
_play(parser, recorder, messages)
|
|
137
|
+
return out.rows
|
|
138
|
+
|
|
139
|
+
|
|
140
|
+
def _tool_turn(name: str, result: str = "60F", is_error: bool = False) -> list:
|
|
141
|
+
return [
|
|
142
|
+
SystemMessage(subtype="init", data={}),
|
|
143
|
+
*_model_call(
|
|
144
|
+
"m1",
|
|
145
|
+
[ToolUseBlock(id="t1", name=name, input={"city": "SF"})],
|
|
146
|
+
20,
|
|
147
|
+
"tool_use",
|
|
148
|
+
),
|
|
149
|
+
UserMessage(
|
|
150
|
+
content=[
|
|
151
|
+
ToolResultBlock(tool_use_id="t1", content=result, is_error=is_error)
|
|
152
|
+
]
|
|
153
|
+
),
|
|
154
|
+
*_model_call("m2", [TextBlock(text="It's foggy.")], 5, "end_turn"),
|
|
155
|
+
_result(),
|
|
156
|
+
]
|
|
157
|
+
|
|
158
|
+
|
|
159
|
+
def _types(rows: list[dict]) -> list[str]:
|
|
160
|
+
return [r["event_type"] for r in rows]
|
|
161
|
+
|
|
162
|
+
|
|
163
|
+
def _content(row: dict):
|
|
164
|
+
return json.loads(row["content"])
|
|
165
|
+
|
|
166
|
+
|
|
167
|
+
def test_a_tool_turn_writes_adks_rows_in_adks_order():
|
|
168
|
+
rows = _record(_tool_turn("mcp__weather__get_weather"))
|
|
169
|
+
|
|
170
|
+
assert _types(rows) == ADK_TOOL_TURN
|
|
171
|
+
assert {r["session_id"] for r in rows} == {"conv-1"}
|
|
172
|
+
assert {r["agent"] for r in rows} == {"weather_agent"}
|
|
173
|
+
assert {r["user_id"] for r in rows} == {"A2A_USER_conv-1"}
|
|
174
|
+
assert len({r["invocation_id"] for r in rows}) == 1
|
|
175
|
+
# The JSON paths ADK's views read.
|
|
176
|
+
assert _content(rows[0]) == {"text_summary": "Weather in SF?"}
|
|
177
|
+
first_response = rows[4]
|
|
178
|
+
usage = _content(first_response)["usage"]
|
|
179
|
+
# Final usage comes from message_delta; prompt includes cached input.
|
|
180
|
+
assert usage == {"prompt": 140, "completion": 20, "total": 160}
|
|
181
|
+
attributes = json.loads(first_response["attributes"])
|
|
182
|
+
assert attributes["finish_reason"] == "tool_use"
|
|
183
|
+
assert attributes["usage_metadata"]["cached_content_token_count"] == 40
|
|
184
|
+
assert "time_to_first_token_ms" in json.loads(first_response["latency_ms"])
|
|
185
|
+
assert _content(rows[5]) == {
|
|
186
|
+
"tool": "mcp__weather__get_weather",
|
|
187
|
+
"args": {"city": "SF"},
|
|
188
|
+
"tool_origin": "MCP",
|
|
189
|
+
}
|
|
190
|
+
assert _content(rows[6])["result"] == "60F"
|
|
191
|
+
assert _content(rows[9]) == {"response": "It's foggy."}
|
|
192
|
+
assert json.loads(rows[11]["attributes"])["session_cost_usd"] == 0.01
|
|
193
|
+
|
|
194
|
+
|
|
195
|
+
@pytest.mark.parametrize(
|
|
196
|
+
("system_prompt", "logged"),
|
|
197
|
+
[
|
|
198
|
+
pytest.param("Be brief.", "Be brief.", id="string"),
|
|
199
|
+
pytest.param(
|
|
200
|
+
{"type": "custom", "prompt": "Be brief."}, "Be brief.", id="custom"
|
|
201
|
+
),
|
|
202
|
+
# The preset's own prompt is the CLI's.
|
|
203
|
+
pytest.param(
|
|
204
|
+
{"type": "preset", "preset": "claude_code", "append": "Be brief."},
|
|
205
|
+
"Be brief.",
|
|
206
|
+
id="preset_with_append",
|
|
207
|
+
),
|
|
208
|
+
pytest.param({"type": "preset", "preset": "claude_code"}, None, id="preset"),
|
|
209
|
+
pytest.param({"type": "file", "path": "prompt.md"}, None, id="file"),
|
|
210
|
+
pytest.param(None, None, id="none"),
|
|
211
|
+
],
|
|
212
|
+
)
|
|
213
|
+
def test_the_request_and_agent_starting_rows_have_the_system_prompts_text(
|
|
214
|
+
system_prompt, logged
|
|
215
|
+
):
|
|
216
|
+
out = FakeWriter()
|
|
217
|
+
parser = TurnParser("a")
|
|
218
|
+
recorder = AgentEventsRecorder(
|
|
219
|
+
out,
|
|
220
|
+
prompt="hi",
|
|
221
|
+
agent_name=parser.agent_name,
|
|
222
|
+
session_id="s",
|
|
223
|
+
user_id="u",
|
|
224
|
+
model=None,
|
|
225
|
+
system_prompt=system_prompt,
|
|
226
|
+
instruction=system_prompt,
|
|
227
|
+
)
|
|
228
|
+
_play(parser, recorder, _tool_turn("Bash"))
|
|
229
|
+
|
|
230
|
+
request = next(r for r in out.rows if r["event_type"] == "LLM_REQUEST")
|
|
231
|
+
assert _content(request).get("system_prompt") == logged
|
|
232
|
+
starting = next(r for r in out.rows if r["event_type"] == "AGENT_STARTING")
|
|
233
|
+
summary = _content(starting)["text_summary"] if starting["content"] else None
|
|
234
|
+
assert summary == logged
|
|
235
|
+
|
|
236
|
+
|
|
237
|
+
def test_rows_join_the_turns_span_once_it_is_current():
|
|
238
|
+
"""The instrumentor's span is current only after query() yields."""
|
|
239
|
+
out = FakeWriter()
|
|
240
|
+
parser = TurnParser("a")
|
|
241
|
+
recorder = AgentEventsRecorder(
|
|
242
|
+
out,
|
|
243
|
+
prompt="hi",
|
|
244
|
+
agent_name=parser.agent_name,
|
|
245
|
+
session_id="s",
|
|
246
|
+
user_id="u",
|
|
247
|
+
model=None,
|
|
248
|
+
system_prompt=None,
|
|
249
|
+
instruction=None,
|
|
250
|
+
)
|
|
251
|
+
tracer = TracerProvider().get_tracer("test")
|
|
252
|
+
with tracer.start_as_current_span("request") as request:
|
|
253
|
+
with tracer.start_as_current_span("ClaudeAgentSDK.query") as turn:
|
|
254
|
+
_play(parser, recorder, [SystemMessage(subtype="init", data={}), _result()])
|
|
255
|
+
|
|
256
|
+
trace_id = format(turn.get_span_context().trace_id, "032x")
|
|
257
|
+
assert {r["trace_id"] for r in out.rows} == {trace_id}
|
|
258
|
+
invocation = next(r for r in out.rows if r["event_type"] == "INVOCATION_STARTING")
|
|
259
|
+
assert invocation["span_id"] == format(turn.get_span_context().span_id, "016x")
|
|
260
|
+
# The request span isn't a row: the root rows have no parent.
|
|
261
|
+
assert request.get_span_context().is_valid
|
|
262
|
+
assert invocation["parent_span_id"] is None
|
|
263
|
+
assert out.rows[-1]["parent_span_id"] is None
|
|
264
|
+
|
|
265
|
+
|
|
266
|
+
def test_failures_become_error_rows():
|
|
267
|
+
rows = _record(_tool_turn("Bash", result="denied", is_error=True))
|
|
268
|
+
tool_error = next(r for r in rows if r["event_type"] == "TOOL_ERROR")
|
|
269
|
+
assert tool_error["status"] == "ERROR"
|
|
270
|
+
assert tool_error["error_message"] == "denied"
|
|
271
|
+
assert _content(tool_error)["tool_origin"] == "LOCAL"
|
|
272
|
+
|
|
273
|
+
out = FakeWriter()
|
|
274
|
+
parser = TurnParser("a")
|
|
275
|
+
recorder = AgentEventsRecorder(
|
|
276
|
+
out,
|
|
277
|
+
prompt="hi",
|
|
278
|
+
agent_name=parser.agent_name,
|
|
279
|
+
session_id="s",
|
|
280
|
+
user_id="u",
|
|
281
|
+
model=None,
|
|
282
|
+
system_prompt=None,
|
|
283
|
+
instruction=None,
|
|
284
|
+
)
|
|
285
|
+
_play(parser, recorder, [], error=RuntimeError("model not available"))
|
|
286
|
+
assert _types(out.rows)[-2:] == ["AGENT_ERROR", "INVOCATION_ERROR"]
|
|
287
|
+
assert out.rows[-1]["error_message"] == "model not available"
|
|
288
|
+
|
|
289
|
+
|
|
290
|
+
def test_a_subagent_gets_its_own_agent_and_model_rows():
|
|
291
|
+
messages = [
|
|
292
|
+
*_model_call(
|
|
293
|
+
"m1",
|
|
294
|
+
[
|
|
295
|
+
ToolUseBlock(
|
|
296
|
+
id="t1",
|
|
297
|
+
name="Agent",
|
|
298
|
+
input={"subagent_type": "researcher", "prompt": "Find it"},
|
|
299
|
+
)
|
|
300
|
+
],
|
|
301
|
+
20,
|
|
302
|
+
"tool_use",
|
|
303
|
+
),
|
|
304
|
+
AssistantMessage(
|
|
305
|
+
content=[TextBlock(text="Found it")],
|
|
306
|
+
model="claude-haiku-4-5",
|
|
307
|
+
message_id="sub-1",
|
|
308
|
+
parent_tool_use_id="t1",
|
|
309
|
+
usage={"input_tokens": 50, "output_tokens": 7},
|
|
310
|
+
),
|
|
311
|
+
UserMessage(content=[ToolResultBlock(tool_use_id="t1", content="Found it")]),
|
|
312
|
+
*_model_call("m2", [TextBlock(text="Done")], 5, "end_turn"),
|
|
313
|
+
_result(),
|
|
314
|
+
]
|
|
315
|
+
rows = _record(messages)
|
|
316
|
+
sub = [r for r in rows if r["agent"] == "researcher"]
|
|
317
|
+
|
|
318
|
+
assert _types(sub) == [
|
|
319
|
+
"AGENT_STARTING",
|
|
320
|
+
"LLM_REQUEST",
|
|
321
|
+
"LLM_RESPONSE",
|
|
322
|
+
"AGENT_COMPLETED",
|
|
323
|
+
]
|
|
324
|
+
tool_start = next(r for r in rows if r["event_type"] == "TOOL_STARTING")
|
|
325
|
+
assert _content(tool_start)["tool_origin"] == "SUB_AGENT"
|
|
326
|
+
# The subagent runs under the subagent tool's span.
|
|
327
|
+
assert sub[0]["parent_span_id"] == tool_start["span_id"]
|
|
328
|
+
assert _content(sub[2])["usage"]["completion"] == 7
|
|
329
|
+
# Its completion comes before the tool's.
|
|
330
|
+
assert _types(rows).index("AGENT_COMPLETED") < _types(rows).index("TOOL_COMPLETED")
|
|
331
|
+
|
|
332
|
+
|
|
333
|
+
@pytest.mark.parametrize(
|
|
334
|
+
("parent", "agent"), [(None, "weather_agent"), ("t1", "researcher")]
|
|
335
|
+
)
|
|
336
|
+
def test_a_failed_model_call_is_an_llm_error_row(parent, agent):
|
|
337
|
+
failed = AssistantMessage(
|
|
338
|
+
content=[TextBlock(text="API Error: 429 rate limited")],
|
|
339
|
+
model="claude-haiku-4-5",
|
|
340
|
+
message_id="bad",
|
|
341
|
+
parent_tool_use_id=parent,
|
|
342
|
+
error="rate_limit",
|
|
343
|
+
)
|
|
344
|
+
if parent is None:
|
|
345
|
+
messages = [failed, _result(is_error=True, result="API Error: 429")]
|
|
346
|
+
else:
|
|
347
|
+
delegate = ToolUseBlock(
|
|
348
|
+
id="t1", name="Agent", input={"subagent_type": "researcher", "prompt": "?"}
|
|
349
|
+
)
|
|
350
|
+
messages = [
|
|
351
|
+
*_model_call("m1", [delegate], 20, "tool_use"),
|
|
352
|
+
failed,
|
|
353
|
+
UserMessage(
|
|
354
|
+
content=[
|
|
355
|
+
ToolResultBlock(tool_use_id="t1", content="429", is_error=True)
|
|
356
|
+
]
|
|
357
|
+
),
|
|
358
|
+
*_model_call("m2", [TextBlock(text="Sorry.")], 5, "end_turn"),
|
|
359
|
+
_result(),
|
|
360
|
+
]
|
|
361
|
+
|
|
362
|
+
rows = _record(messages)
|
|
363
|
+
|
|
364
|
+
[error] = [r for r in rows if r["event_type"] == "LLM_ERROR"]
|
|
365
|
+
assert error["agent"] == agent
|
|
366
|
+
assert error["status"] == "ERROR"
|
|
367
|
+
assert error["error_message"] == "rate_limit: API Error: 429 rate limited"
|
|
368
|
+
assert not [
|
|
369
|
+
r for r in rows if r["event_type"] == "LLM_RESPONSE" and r["agent"] == agent
|
|
370
|
+
]
|
|
371
|
+
|
|
372
|
+
|
|
373
|
+
def _assistant(message_id: str, block, parent: str | None = None):
|
|
374
|
+
return AssistantMessage(
|
|
375
|
+
content=[block],
|
|
376
|
+
model="claude-haiku-4-5",
|
|
377
|
+
message_id=message_id,
|
|
378
|
+
parent_tool_use_id=parent,
|
|
379
|
+
usage={"input_tokens": 100, "output_tokens": 6},
|
|
380
|
+
)
|
|
381
|
+
|
|
382
|
+
|
|
383
|
+
def test_a_background_subagent_ends_at_its_task_notification():
|
|
384
|
+
"""The CLI returns the tool before the stream ends; the subagent's report
|
|
385
|
+
starts another round after the first result."""
|
|
386
|
+
task = {"tool_use_id": "t1", "subagent_type": "forecaster", "task_id": "a9"}
|
|
387
|
+
call = {"subagent_type": "forecaster", "prompt": "Weather in Tokyo?"}
|
|
388
|
+
messages = [
|
|
389
|
+
SystemMessage(subtype="init", data={}),
|
|
390
|
+
_event("message_start", message={"usage": {"input_tokens": 3777}}),
|
|
391
|
+
_assistant("m1", ToolUseBlock(id="t1", name="Agent", input=call)),
|
|
392
|
+
SystemMessage(subtype="task_started", data=task),
|
|
393
|
+
_event(
|
|
394
|
+
"message_delta",
|
|
395
|
+
usage={"output_tokens": 189},
|
|
396
|
+
delta={"stop_reason": "tool_use"},
|
|
397
|
+
),
|
|
398
|
+
UserMessage(
|
|
399
|
+
content=[ToolResultBlock(tool_use_id="t1", content="Async agent launched")]
|
|
400
|
+
),
|
|
401
|
+
_event("message_stop"),
|
|
402
|
+
*_model_call("m2", [TextBlock(text="Delegated.")], 33, "end_turn"),
|
|
403
|
+
_result(result="Delegated."),
|
|
404
|
+
_assistant(
|
|
405
|
+
"s1",
|
|
406
|
+
ToolUseBlock(id="t2", name="mcp__weather__get_weather", input={}),
|
|
407
|
+
parent="t1",
|
|
408
|
+
),
|
|
409
|
+
SystemMessage(subtype="task_progress", data=task),
|
|
410
|
+
UserMessage(
|
|
411
|
+
content=[ToolResultBlock(tool_use_id="t2", content="Sunny")],
|
|
412
|
+
parent_tool_use_id="t1",
|
|
413
|
+
),
|
|
414
|
+
_assistant("s2", TextBlock(text="Tokyo is sunny."), parent="t1"),
|
|
415
|
+
SystemMessage(subtype="task_updated", data={"task_id": "a9"}),
|
|
416
|
+
SystemMessage(
|
|
417
|
+
subtype="task_notification",
|
|
418
|
+
data={"tool_use_id": "t1", "status": "completed", "task_id": "a9"},
|
|
419
|
+
),
|
|
420
|
+
SystemMessage(subtype="init", data={}),
|
|
421
|
+
*_model_call("m3", [TextBlock(text="It's sunny in Tokyo.")], 90, "end_turn"),
|
|
422
|
+
_result(result="It's sunny in Tokyo."),
|
|
423
|
+
]
|
|
424
|
+
rows = _record(messages)
|
|
425
|
+
types = _types(rows)
|
|
426
|
+
|
|
427
|
+
sub = [r for r in rows if r["agent"] == "forecaster"]
|
|
428
|
+
assert _types(sub) == [
|
|
429
|
+
"AGENT_STARTING",
|
|
430
|
+
"LLM_REQUEST",
|
|
431
|
+
"LLM_RESPONSE",
|
|
432
|
+
"TOOL_STARTING",
|
|
433
|
+
"TOOL_COMPLETED",
|
|
434
|
+
"LLM_REQUEST",
|
|
435
|
+
"LLM_RESPONSE",
|
|
436
|
+
"AGENT_COMPLETED",
|
|
437
|
+
]
|
|
438
|
+
tool_start = next(r for r in rows if r["event_type"] == "TOOL_STARTING")
|
|
439
|
+
assert sub[0]["parent_span_id"] == tool_start["span_id"]
|
|
440
|
+
# The launch's LLM_RESPONSE has the stream's final usage.
|
|
441
|
+
assert _content(rows[types.index("LLM_RESPONSE")])["usage"]["completion"] == 189
|
|
442
|
+
# Every model call, including the round after the first result, is inside
|
|
443
|
+
# the turn, which closes with the last result.
|
|
444
|
+
assert types.count("LLM_RESPONSE") == 5
|
|
445
|
+
assert types[-3:] == ["AGENT_RESPONSE", "AGENT_COMPLETED", "INVOCATION_COMPLETED"]
|
|
446
|
+
assert _content(rows[-3]) == {"response": "It's sunny in Tokyo."}
|
|
447
|
+
|
|
448
|
+
|
|
449
|
+
@pytest.mark.parametrize(
|
|
450
|
+
("result", "turn_end"),
|
|
451
|
+
[
|
|
452
|
+
(_result(), ["AGENT_RESPONSE", "AGENT_COMPLETED", "INVOCATION_COMPLETED"]),
|
|
453
|
+
(_result(is_error=True), ["AGENT_ERROR", "INVOCATION_ERROR"]),
|
|
454
|
+
],
|
|
455
|
+
)
|
|
456
|
+
def test_a_subagent_still_running_at_the_end_of_the_turn_is_closed(result, turn_end):
|
|
457
|
+
"""A background subagent whose task_notification never came."""
|
|
458
|
+
call = {"subagent_type": "researcher", "prompt": "Find it"}
|
|
459
|
+
messages = [
|
|
460
|
+
SystemMessage(subtype="init", data={}),
|
|
461
|
+
_event("message_start", message={"usage": {"input_tokens": 100}}),
|
|
462
|
+
_assistant("m1", ToolUseBlock(id="t1", name="Agent", input=call)),
|
|
463
|
+
SystemMessage(subtype="task_started", data={"tool_use_id": "t1"}),
|
|
464
|
+
UserMessage(
|
|
465
|
+
content=[ToolResultBlock(tool_use_id="t1", content="Async agent launched")]
|
|
466
|
+
),
|
|
467
|
+
_event("message_stop"),
|
|
468
|
+
_assistant(
|
|
469
|
+
"s1",
|
|
470
|
+
ToolUseBlock(id="t2", name="mcp__weather__get_weather", input={}),
|
|
471
|
+
parent="t1",
|
|
472
|
+
),
|
|
473
|
+
result,
|
|
474
|
+
]
|
|
475
|
+
rows = _record(messages)
|
|
476
|
+
|
|
477
|
+
sub = [r for r in rows if r["agent"] == "researcher"]
|
|
478
|
+
assert _types(sub) == [
|
|
479
|
+
"AGENT_STARTING",
|
|
480
|
+
"LLM_REQUEST",
|
|
481
|
+
"LLM_RESPONSE",
|
|
482
|
+
"TOOL_STARTING",
|
|
483
|
+
turn_end[-2],
|
|
484
|
+
]
|
|
485
|
+
# Closed just before the turn is, with the turn's outcome.
|
|
486
|
+
assert rows[-len(turn_end) - 1] is sub[-1]
|
|
487
|
+
assert _types(rows)[-len(turn_end) :] == turn_end
|
|
488
|
+
|
|
489
|
+
|
|
490
|
+
def test_a_tool_result_before_the_stream_ends_waits_for_the_final_usage():
|
|
491
|
+
"""A denied call's result comes before the stream's message_delta."""
|
|
492
|
+
messages = [
|
|
493
|
+
SystemMessage(subtype="init", data={}),
|
|
494
|
+
_event("message_start", message={"usage": {"input_tokens": 3772}}),
|
|
495
|
+
_assistant(
|
|
496
|
+
"m1", ToolUseBlock(id="t1", name="mcp__weather__get_time", input={})
|
|
497
|
+
),
|
|
498
|
+
SystemMessage(subtype="permission_denied", data={"tool_use_id": "t1"}),
|
|
499
|
+
UserMessage(
|
|
500
|
+
content=[ToolResultBlock(tool_use_id="t1", content="denied", is_error=True)]
|
|
501
|
+
),
|
|
502
|
+
_event(
|
|
503
|
+
"message_delta",
|
|
504
|
+
usage={"output_tokens": 101},
|
|
505
|
+
delta={"stop_reason": "tool_use"},
|
|
506
|
+
),
|
|
507
|
+
_event("message_stop"),
|
|
508
|
+
*_model_call("m2", [TextBlock(text="I can't.")], 122, "end_turn"),
|
|
509
|
+
_result(result="I can't."),
|
|
510
|
+
]
|
|
511
|
+
rows = _record(messages)
|
|
512
|
+
|
|
513
|
+
assert _types(rows)[3:7] == [
|
|
514
|
+
"LLM_REQUEST",
|
|
515
|
+
"LLM_RESPONSE",
|
|
516
|
+
"TOOL_STARTING",
|
|
517
|
+
"TOOL_ERROR",
|
|
518
|
+
]
|
|
519
|
+
assert _content(rows[4])["usage"]["completion"] == 101
|
|
520
|
+
assert json.loads(rows[4]["attributes"])["finish_reason"] == "tool_use"
|
|
521
|
+
|
|
522
|
+
|
|
523
|
+
def test_a_turn_stopped_at_an_error_result_records_the_result():
|
|
524
|
+
out = FakeWriter()
|
|
525
|
+
parser = TurnParser("a")
|
|
526
|
+
recorder = AgentEventsRecorder(
|
|
527
|
+
out,
|
|
528
|
+
prompt="hi",
|
|
529
|
+
agent_name=parser.agent_name,
|
|
530
|
+
session_id="s",
|
|
531
|
+
user_id="u",
|
|
532
|
+
model=None,
|
|
533
|
+
system_prompt=None,
|
|
534
|
+
instruction=None,
|
|
535
|
+
)
|
|
536
|
+
# run_turn stopped reading at the result.
|
|
537
|
+
_play(
|
|
538
|
+
parser,
|
|
539
|
+
recorder,
|
|
540
|
+
[_result(is_error=True, result="Reached max turns")],
|
|
541
|
+
error=GeneratorExit(),
|
|
542
|
+
)
|
|
543
|
+
assert out.rows[-1]["event_type"] == "INVOCATION_ERROR"
|
|
544
|
+
assert out.rows[-1]["error_message"] == "Reached max turns"
|
|
545
|
+
|
|
546
|
+
|
|
547
|
+
def test_a_remote_agent_call_adds_an_a2a_interaction():
|
|
548
|
+
rows = _record(
|
|
549
|
+
_tool_turn("mcp__weather__ask", result="Sunny"),
|
|
550
|
+
remote_agents={"weather": "https://weather.example/a2a/app"},
|
|
551
|
+
)
|
|
552
|
+
started = next(r for r in rows if r["event_type"] == "TOOL_STARTING")
|
|
553
|
+
assert _content(started)["tool_origin"] == "A2A"
|
|
554
|
+
interaction = next(r for r in rows if r["event_type"] == "A2A_INTERACTION")
|
|
555
|
+
metadata = json.loads(interaction["attributes"])["a2a_metadata"]
|
|
556
|
+
assert metadata["a2a:url"] == "https://weather.example/a2a/app"
|
|
557
|
+
assert metadata["a2a:response"] == "Sunny"
|
|
558
|
+
|
|
559
|
+
|
|
560
|
+
def test_a_compaction_is_recorded():
|
|
561
|
+
rows = _record(
|
|
562
|
+
[
|
|
563
|
+
SystemMessage(
|
|
564
|
+
subtype="compact_boundary",
|
|
565
|
+
data={"compact_metadata": {"trigger": "auto", "pre_tokens": 150000}},
|
|
566
|
+
),
|
|
567
|
+
_result(),
|
|
568
|
+
]
|
|
569
|
+
)
|
|
570
|
+
compaction = next(r for r in rows if r["event_type"] == "EVENT_COMPACTION")
|
|
571
|
+
assert json.loads(compaction["attributes"])["trigger"] == "auto"
|
|
572
|
+
|
|
573
|
+
|
|
574
|
+
def test_a_malformed_subagent_call_is_still_recorded():
|
|
575
|
+
odd = AssistantMessage(
|
|
576
|
+
content=[ToolUseBlock(id="t1", name="Agent", input=["not", "a", "dict"])],
|
|
577
|
+
model="m",
|
|
578
|
+
)
|
|
579
|
+
rows = _record([odd, _event("message_stop"), _result()]) # must not raise
|
|
580
|
+
assert "INVOCATION_COMPLETED" in _types(rows)
|
|
581
|
+
started = next(
|
|
582
|
+
r
|
|
583
|
+
for r in rows
|
|
584
|
+
if r["event_type"] == "AGENT_STARTING" and r["agent"] != "weather_agent"
|
|
585
|
+
)
|
|
586
|
+
assert started["agent"] == "subagent"
|
|
587
|
+
|
|
588
|
+
|
|
589
|
+
def test_tool_rows_are_redacted_and_capped():
|
|
590
|
+
rows = _record(
|
|
591
|
+
_tool_turn("mcp__weather__ask", result="x" * (MAX_CONTENT_LENGTH + 10)),
|
|
592
|
+
remote_agents={"weather": "https://weather.example/a2a/app"},
|
|
593
|
+
)
|
|
594
|
+
|
|
595
|
+
completed = next(r for r in rows if r["event_type"] == "TOOL_COMPLETED")
|
|
596
|
+
assert completed["is_truncated"] is True
|
|
597
|
+
assert _content(completed)["result"].endswith("...[TRUNCATED]")
|
|
598
|
+
interaction = next(r for r in rows if r["event_type"] == "A2A_INTERACTION")
|
|
599
|
+
metadata = json.loads(interaction["attributes"])["a2a_metadata"]
|
|
600
|
+
assert interaction["is_truncated"] is True
|
|
601
|
+
assert metadata["a2a:response"].endswith("...[TRUNCATED]")
|
|
602
|
+
|
|
603
|
+
|
|
604
|
+
def test_tool_arguments_have_credentials_redacted():
|
|
605
|
+
turn = _tool_turn("mcp__crm__lookup")
|
|
606
|
+
for message in turn:
|
|
607
|
+
for block in getattr(message, "content", None) or []:
|
|
608
|
+
if isinstance(block, ToolUseBlock):
|
|
609
|
+
block.input["api_key"] = "sk-live-123"
|
|
610
|
+
rows = _record(turn)
|
|
611
|
+
|
|
612
|
+
started = next(r for r in rows if r["event_type"] == "TOOL_STARTING")
|
|
613
|
+
assert _content(started)["args"] == {"city": "SF", "api_key": "[REDACTED]"}
|
|
614
|
+
assert "sk-live-123" not in json.dumps(rows)
|
|
615
|
+
|
|
616
|
+
|
|
617
|
+
def test_long_content_is_truncated_at_adks_limit():
|
|
618
|
+
rows = _record([_result(result="x" * (MAX_CONTENT_LENGTH + 10))])
|
|
619
|
+
response = next(r for r in rows if r["event_type"] == "AGENT_RESPONSE")
|
|
620
|
+
assert response["is_truncated"] is True
|
|
621
|
+
assert _content(response)["response"].endswith("...[TRUNCATED]")
|
|
622
|
+
|
|
623
|
+
|
|
624
|
+
@pytest.fixture
|
|
625
|
+
def fresh_writer(monkeypatch):
|
|
626
|
+
monkeypatch.setattr(bq_analytics, "_writer", None)
|
|
627
|
+
monkeypatch.setattr(bq_analytics, "_writer_failed", False)
|
|
628
|
+
monkeypatch.setenv("GOOGLE_CLOUD_PROJECT", "proj")
|
|
629
|
+
|
|
630
|
+
|
|
631
|
+
def test_no_dataset_means_no_writer(fresh_writer, monkeypatch):
|
|
632
|
+
monkeypatch.delenv("BQ_ANALYTICS_DATASET_ID", raising=False)
|
|
633
|
+
with patch("google.cloud.bigquery.Client") as client:
|
|
634
|
+
assert bq_analytics.writer() is None
|
|
635
|
+
client.assert_not_called()
|
|
636
|
+
|
|
637
|
+
|
|
638
|
+
@patch("google.cloud.bigquery.Client")
|
|
639
|
+
def test_the_writer_creates_adks_table_and_views_then_streams_rows(
|
|
640
|
+
client_cls, fresh_writer, monkeypatch
|
|
641
|
+
):
|
|
642
|
+
monkeypatch.setenv("BQ_ANALYTICS_DATASET_ID", "telemetry")
|
|
643
|
+
client = client_cls.return_value
|
|
644
|
+
client.get_table.side_effect = NotFound("no table")
|
|
645
|
+
client.list_tables.return_value = []
|
|
646
|
+
client.insert_rows_json.return_value = []
|
|
647
|
+
|
|
648
|
+
out = bq_analytics.writer()
|
|
649
|
+
assert out is not None
|
|
650
|
+
out.put({"event_id": "e-1", "event_type": "INVOCATION_STARTING"})
|
|
651
|
+
assert out.flush(5)
|
|
652
|
+
|
|
653
|
+
table, *views = [c.args[0] for c in client.create_table.call_args_list]
|
|
654
|
+
assert table.table_id == agent_events.TABLE
|
|
655
|
+
assert table.time_partitioning.field == "timestamp"
|
|
656
|
+
assert table.clustering_fields == agent_events.CLUSTERING_FIELDS
|
|
657
|
+
assert [f.name for f in table.schema] == [
|
|
658
|
+
f["name"] for f in agent_events.EVENTS_SCHEMA
|
|
659
|
+
]
|
|
660
|
+
assert len(views) == len(agent_events.EVENT_VIEW_DEFS)
|
|
661
|
+
llm_response = next(v for v in views if v.table_id == "v_llm_response")
|
|
662
|
+
assert "FROM\n `proj.telemetry.agent_events`" in llm_response.view_query
|
|
663
|
+
assert "event_type = 'LLM_RESPONSE'" in llm_response.view_query
|
|
664
|
+
client.insert_rows_json.assert_called_once_with(
|
|
665
|
+
"proj.telemetry.agent_events",
|
|
666
|
+
[{"event_id": "e-1", "event_type": "INVOCATION_STARTING"}],
|
|
667
|
+
row_ids=["e-1"],
|
|
668
|
+
retry=ANY,
|
|
669
|
+
timeout=ANY,
|
|
670
|
+
)
|
|
671
|
+
|
|
672
|
+
|
|
673
|
+
def _listed(views) -> list[TableListItem]:
|
|
674
|
+
"""What list_tables returns for these views."""
|
|
675
|
+
return [
|
|
676
|
+
TableListItem(
|
|
677
|
+
{
|
|
678
|
+
"tableReference": {
|
|
679
|
+
"projectId": "proj",
|
|
680
|
+
"datasetId": "telemetry",
|
|
681
|
+
"tableId": v.table_id,
|
|
682
|
+
},
|
|
683
|
+
"type": "VIEW",
|
|
684
|
+
"labels": v.labels,
|
|
685
|
+
}
|
|
686
|
+
)
|
|
687
|
+
for v in views
|
|
688
|
+
]
|
|
689
|
+
|
|
690
|
+
|
|
691
|
+
@patch("google.cloud.bigquery.Client")
|
|
692
|
+
def test_only_missing_or_outdated_views_are_written(
|
|
693
|
+
client_cls, fresh_writer, monkeypatch
|
|
694
|
+
):
|
|
695
|
+
monkeypatch.setenv("BQ_ANALYTICS_DATASET_ID", "telemetry")
|
|
696
|
+
client = client_cls.return_value
|
|
697
|
+
client.list_tables.return_value = []
|
|
698
|
+
assert bq_analytics.writer().flush(5)
|
|
699
|
+
created = [c.args[0] for c in client.create_table.call_args_list]
|
|
700
|
+
|
|
701
|
+
# The next start finds them all current except one ADK changed, and one
|
|
702
|
+
# missing.
|
|
703
|
+
monkeypatch.setattr(bq_analytics, "_writer", None)
|
|
704
|
+
client.reset_mock()
|
|
705
|
+
current, outdated, _missing = created[:-2], created[-2], created[-1]
|
|
706
|
+
outdated.labels = {**outdated.labels, tables._VIEW_HASH_LABEL: "old"}
|
|
707
|
+
client.list_tables.return_value = _listed([*current, outdated])
|
|
708
|
+
assert bq_analytics.writer().flush(5)
|
|
709
|
+
|
|
710
|
+
assert [c.args[0].table_id for c in client.create_table.call_args_list] == [
|
|
711
|
+
created[-1].table_id
|
|
712
|
+
]
|
|
713
|
+
(update,) = client.update_table.call_args_list
|
|
714
|
+
assert update.args[0].table_id == outdated.table_id
|
|
715
|
+
assert update.args[1] == ["view_query", "labels"]
|
|
716
|
+
|
|
717
|
+
|
|
718
|
+
@patch("google.cloud.bigquery.Client")
|
|
719
|
+
def test_a_table_it_cant_set_up_is_reported_once(
|
|
720
|
+
client_cls, fresh_writer, monkeypatch, caplog
|
|
721
|
+
):
|
|
722
|
+
monkeypatch.setenv("BQ_ANALYTICS_DATASET_ID", "telemetry")
|
|
723
|
+
client = client_cls.return_value
|
|
724
|
+
client.get_table.side_effect = NotFound("Not found: Dataset")
|
|
725
|
+
client.create_table.side_effect = NotFound("Not found: Dataset")
|
|
726
|
+
client.insert_rows_json.side_effect = NotFound("Not found: Table")
|
|
727
|
+
|
|
728
|
+
out = bq_analytics.writer()
|
|
729
|
+
for n in range(3): # one batch each
|
|
730
|
+
out.put({"event_id": f"e-{n}"})
|
|
731
|
+
assert out.flush(5)
|
|
732
|
+
|
|
733
|
+
assert client.insert_rows_json.call_count == 3
|
|
734
|
+
assert len([r for r in caplog.records if r.levelname == "WARNING"]) == 1
|
|
735
|
+
client.list_tables.assert_not_called()
|
|
736
|
+
|
|
737
|
+
|
|
738
|
+
@patch("google.cloud.bigquery.Client")
|
|
739
|
+
def test_rows_still_queued_at_shutdown_are_reported(
|
|
740
|
+
client_cls, fresh_writer, monkeypatch, caplog
|
|
741
|
+
):
|
|
742
|
+
monkeypatch.setenv("BQ_ANALYTICS_DATASET_ID", "telemetry")
|
|
743
|
+
release = threading.Event()
|
|
744
|
+
client = client_cls.return_value
|
|
745
|
+
client.get_table.side_effect = lambda _: release.wait(5)
|
|
746
|
+
|
|
747
|
+
out = bq_analytics.writer()
|
|
748
|
+
out.put({"event_id": "e-1"})
|
|
749
|
+
assert not out.flush(0.1)
|
|
750
|
+
release.set()
|
|
751
|
+
|
|
752
|
+
assert "1 rows weren't written" in caplog.text
|
|
753
|
+
|
|
754
|
+
|
|
755
|
+
@patch("google.cloud.bigquery.Client")
|
|
756
|
+
def test_rows_past_the_queue_bound_are_dropped_not_held(
|
|
757
|
+
client_cls, fresh_writer, monkeypatch, caplog
|
|
758
|
+
):
|
|
759
|
+
monkeypatch.setenv("BQ_ANALYTICS_DATASET_ID", "telemetry")
|
|
760
|
+
monkeypatch.setattr(uploader, "_MAX_QUEUED_ROWS", 2)
|
|
761
|
+
release = threading.Event()
|
|
762
|
+
client = client_cls.return_value
|
|
763
|
+
client.get_table.side_effect = lambda _: release.wait(5)
|
|
764
|
+
client.insert_rows_json.return_value = []
|
|
765
|
+
|
|
766
|
+
out = bq_analytics.writer()
|
|
767
|
+
for n in range(4):
|
|
768
|
+
out.put({"event_id": f"e-{n}"})
|
|
769
|
+
release.set()
|
|
770
|
+
assert out.flush(5)
|
|
771
|
+
|
|
772
|
+
rows = [r for c in client.insert_rows_json.call_args_list for r in c.args[1]]
|
|
773
|
+
assert [r["event_id"] for r in rows] == ["e-0", "e-1"]
|
|
774
|
+
assert caplog.text.count("new rows are dropped") == 1
|
|
775
|
+
|
|
776
|
+
|
|
777
|
+
@patch("google.cloud.bigquery.Client")
|
|
778
|
+
def test_a_batch_is_split_under_the_request_size_limit(
|
|
779
|
+
client_cls, fresh_writer, monkeypatch
|
|
780
|
+
):
|
|
781
|
+
monkeypatch.setenv("BQ_ANALYTICS_DATASET_ID", "telemetry")
|
|
782
|
+
release = threading.Event()
|
|
783
|
+
client = client_cls.return_value
|
|
784
|
+
client.get_table.side_effect = lambda _: release.wait(5)
|
|
785
|
+
client.insert_rows_json.return_value = []
|
|
786
|
+
|
|
787
|
+
out = bq_analytics.writer()
|
|
788
|
+
# Queued together, as a busy agent's rows are: 30 turns of 450 KB prompts.
|
|
789
|
+
for n in range(60):
|
|
790
|
+
out.put({"event_id": f"e-{n}", "content": "x" * 450 * 1024})
|
|
791
|
+
release.set()
|
|
792
|
+
assert out.flush(5)
|
|
793
|
+
|
|
794
|
+
calls = client.insert_rows_json.call_args_list
|
|
795
|
+
assert len(calls) > 1
|
|
796
|
+
for call in calls:
|
|
797
|
+
body = {"rows": [{"insertId": r["event_id"], "json": r} for r in call.args[1]]}
|
|
798
|
+
assert len(json.dumps(body)) < 10 * 1024 * 1024 # insertAll's limit
|
|
799
|
+
rows = [r["event_id"] for c in calls for r in c.args[1]]
|
|
800
|
+
assert rows == [f"e-{n}" for n in range(60)]
|
|
801
|
+
|
|
802
|
+
|
|
803
|
+
@patch("google.cloud.bigquery.Client")
|
|
804
|
+
def test_a_write_gives_up_after_a_bounded_time(client_cls, fresh_writer, monkeypatch):
|
|
805
|
+
"""A slow BigQuery mustn't hold the writer for the default 10 minutes."""
|
|
806
|
+
monkeypatch.setenv("BQ_ANALYTICS_DATASET_ID", "telemetry")
|
|
807
|
+
client = client_cls.return_value
|
|
808
|
+
client.insert_rows_json.return_value = []
|
|
809
|
+
|
|
810
|
+
out = bq_analytics.writer()
|
|
811
|
+
out.put({"event_id": "e-1"})
|
|
812
|
+
assert out.flush(5)
|
|
813
|
+
|
|
814
|
+
kwargs = client.insert_rows_json.call_args.kwargs
|
|
815
|
+
assert 0 < kwargs["timeout"] <= 60
|
|
816
|
+
assert 0 < kwargs["retry"].timeout <= 60
|
|
817
|
+
|
|
818
|
+
|
|
819
|
+
@patch("google.cloud.bigquery.Client")
|
|
820
|
+
def test_the_agents_own_tools_are_local(client_cls, fresh_writer, monkeypatch):
|
|
821
|
+
"""In-process @tool functions are ADK's function tools; others are MCP."""
|
|
822
|
+
monkeypatch.setenv("BQ_ANALYTICS_DATASET_ID", "telemetry")
|
|
823
|
+
client = client_cls.return_value
|
|
824
|
+
client.insert_rows_json.return_value = []
|
|
825
|
+
options = ClaudeAgentOptions(
|
|
826
|
+
mcp_servers={
|
|
827
|
+
"weather": create_sdk_mcp_server(name="weather", tools=[]),
|
|
828
|
+
"docs": {"type": "http", "url": "https://docs.example/mcp"},
|
|
829
|
+
}
|
|
830
|
+
)
|
|
831
|
+
|
|
832
|
+
for name in ("mcp__weather__get_weather", "mcp__docs__search"):
|
|
833
|
+
parser = TurnParser(agent_name())
|
|
834
|
+
recorder = bq_analytics.start_turn(
|
|
835
|
+
"hi", options, "conv-1", "conv-1", None, parser.agent_name, instruction=None
|
|
836
|
+
)
|
|
837
|
+
assert recorder is not None
|
|
838
|
+
_play(parser, recorder, _tool_turn(name))
|
|
839
|
+
assert bq_analytics.writer().flush(5)
|
|
840
|
+
|
|
841
|
+
rows = [r for c in client.insert_rows_json.call_args_list for r in c.args[1]]
|
|
842
|
+
origins = [
|
|
843
|
+
_content(r)["tool_origin"] for r in rows if r["event_type"] == "TOOL_STARTING"
|
|
844
|
+
]
|
|
845
|
+
assert origins == ["LOCAL", "MCP"]
|
|
846
|
+
|
|
847
|
+
|
|
848
|
+
@patch("google.cloud.bigquery.Client")
|
|
849
|
+
def test_a_remote_agent_servers_tool_is_an_a2a_call(
|
|
850
|
+
client_cls, fresh_writer, monkeypatch
|
|
851
|
+
):
|
|
852
|
+
monkeypatch.setenv("BQ_ANALYTICS_DATASET_ID", "telemetry")
|
|
853
|
+
client = client_cls.return_value
|
|
854
|
+
client.insert_rows_json.return_value = []
|
|
855
|
+
url = "https://billing.example/a2a/app"
|
|
856
|
+
options = ClaudeAgentOptions(
|
|
857
|
+
mcp_servers={
|
|
858
|
+
"billing": remote_agent("billing", url),
|
|
859
|
+
"docs": {"type": "http", "url": "https://docs.example/mcp"},
|
|
860
|
+
}
|
|
861
|
+
)
|
|
862
|
+
|
|
863
|
+
parser = TurnParser(agent_name())
|
|
864
|
+
recorder = bq_analytics.start_turn(
|
|
865
|
+
"hi", options, "conv-1", "conv-1", None, parser.agent_name, instruction=None
|
|
866
|
+
)
|
|
867
|
+
assert recorder is not None
|
|
868
|
+
_play(parser, recorder, _tool_turn("mcp__billing__ask"))
|
|
869
|
+
assert bq_analytics.writer().flush(5)
|
|
870
|
+
|
|
871
|
+
rows = [r for c in client.insert_rows_json.call_args_list for r in c.args[1]]
|
|
872
|
+
interaction = next(r for r in rows if r["event_type"] == "A2A_INTERACTION")
|
|
873
|
+
assert json.loads(interaction["attributes"])["a2a_metadata"]["a2a:url"] == url
|
|
874
|
+
|
|
875
|
+
|
|
876
|
+
@patch("google.cloud.bigquery.Client")
|
|
877
|
+
def test_an_unexpected_write_error_doesnt_stop_the_writer(
|
|
878
|
+
client_cls, fresh_writer, monkeypatch, caplog
|
|
879
|
+
):
|
|
880
|
+
monkeypatch.setenv("BQ_ANALYTICS_DATASET_ID", "telemetry")
|
|
881
|
+
client = client_cls.return_value
|
|
882
|
+
client.insert_rows_json.return_value = []
|
|
883
|
+
circular: dict = {"event_id": "e-1"}
|
|
884
|
+
circular["self"] = circular
|
|
885
|
+
|
|
886
|
+
out = bq_analytics.writer()
|
|
887
|
+
out.put(circular)
|
|
888
|
+
assert out.flush(5)
|
|
889
|
+
out.put({"event_id": "e-2"})
|
|
890
|
+
assert out.flush(5)
|
|
891
|
+
|
|
892
|
+
rows = [r for c in client.insert_rows_json.call_args_list for r in c.args[1]]
|
|
893
|
+
assert [r["event_id"] for r in rows] == ["e-2"]
|
|
894
|
+
assert "Circular reference" in caplog.text
|
|
895
|
+
|
|
896
|
+
|
|
897
|
+
@patch("google.cloud.bigquery.Client")
|
|
898
|
+
def test_rows_name_the_agent_after_the_setting(client_cls, fresh_writer, monkeypatch):
|
|
899
|
+
monkeypatch.setenv("BQ_ANALYTICS_DATASET_ID", "telemetry")
|
|
900
|
+
monkeypatch.setattr(settings, "AGENT_NAME", "weather_agent")
|
|
901
|
+
client_cls.return_value.insert_rows_json.return_value = []
|
|
902
|
+
|
|
903
|
+
parser = TurnParser(agent_name())
|
|
904
|
+
recorder = bq_analytics.start_turn(
|
|
905
|
+
"hi",
|
|
906
|
+
ClaudeAgentOptions(),
|
|
907
|
+
"conv-1",
|
|
908
|
+
"conv-1",
|
|
909
|
+
None,
|
|
910
|
+
parser.agent_name,
|
|
911
|
+
instruction=None,
|
|
912
|
+
)
|
|
913
|
+
assert recorder is not None
|
|
914
|
+
_play(parser, recorder, _tool_turn("mcp__weather__get_weather"))
|
|
915
|
+
assert bq_analytics.writer().flush(5)
|
|
916
|
+
|
|
917
|
+
calls = client_cls.return_value.insert_rows_json.call_args_list
|
|
918
|
+
rows = [r for c in calls for r in c.args[1]]
|
|
919
|
+
assert rows and {r["agent"] for r in rows} == {"weather_agent"}
|
|
920
|
+
|
|
921
|
+
|
|
922
|
+
def test_missing_credentials_turn_analytics_off(fresh_writer, monkeypatch):
|
|
923
|
+
monkeypatch.setenv("BQ_ANALYTICS_DATASET_ID", "telemetry")
|
|
924
|
+
with patch(
|
|
925
|
+
"google.cloud.bigquery.Client", MagicMock(side_effect=OSError("no ADC"))
|
|
926
|
+
):
|
|
927
|
+
assert bq_analytics.writer() is None
|
|
928
|
+
assert bq_analytics.writer() is None # not retried every turn
|