pytest-copilot 2.0.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- pytest_copilot/__init__.py +133 -0
- pytest_copilot/copilot/__init__.py +47 -0
- pytest_copilot/copilot/_command_job.py +109 -0
- pytest_copilot/copilot/api.py +12 -0
- pytest_copilot/copilot/client.py +58 -0
- pytest_copilot/copilot/commands.py +265 -0
- pytest_copilot/copilot/config.py +133 -0
- pytest_copilot/copilot/contracts.py +222 -0
- pytest_copilot/copilot/controls.py +230 -0
- pytest_copilot/copilot/events.py +795 -0
- pytest_copilot/copilot/fixtures.py +289 -0
- pytest_copilot/copilot/personas.py +607 -0
- pytest_copilot/copilot/requests.py +350 -0
- pytest_copilot/copilot/result.py +309 -0
- pytest_copilot/copilot/runner.py +266 -0
- pytest_copilot/copilot/skills.py +104 -0
- pytest_copilot/copilot/task.py +546 -0
- pytest_copilot/core/__init__.py +77 -0
- pytest_copilot/core/errors.py +44 -0
- pytest_copilot/core/evals.py +521 -0
- pytest_copilot/core/plugin.py +403 -0
- pytest_copilot/core/prompt.py +167 -0
- pytest_copilot/core/result.py +543 -0
- pytest_copilot/core/serialization.py +128 -0
- pytest_copilot/core/skill.py +406 -0
- pytest_copilot/core/skill_evals.py +148 -0
- pytest_copilot/core/skill_grading.py +78 -0
- pytest_copilot/core/validation.py +83 -0
- pytest_copilot/execution/__init__.py +15 -0
- pytest_copilot/execution/cost.py +159 -0
- pytest_copilot/execution/rate_limiter.py +177 -0
- pytest_copilot/execution/servers.py +442 -0
- pytest_copilot/execution/skill_tools.py +228 -0
- pytest_copilot/fixtures/__init__.py +18 -0
- pytest_copilot/fixtures/factories.py +44 -0
- pytest_copilot/fixtures/iteration.py +16 -0
- pytest_copilot/onboarding.py +413 -0
- pytest_copilot/plugin.py +562 -0
- pytest_copilot/plugin_options.py +32 -0
- pytest_copilot/plugin_report.py +18 -0
- pytest_copilot/reporting/__init__.py +16 -0
- pytest_copilot/reporting/collector.py +203 -0
- pytest_copilot/reporting/generator.py +55 -0
- pytest_copilot/reporting/schema.py +5 -0
- pytest_copilot/testing/__init__.py +17 -0
- pytest_copilot/testing/banking.py +479 -0
- pytest_copilot/testing/banking_mcp.py +167 -0
- pytest_copilot/testing/todo.py +380 -0
- pytest_copilot/testing/todo_mcp.py +156 -0
- pytest_copilot/testing/types.py +31 -0
- pytest_copilot-2.0.0.dist-info/METADATA +177 -0
- pytest_copilot-2.0.0.dist-info/RECORD +55 -0
- pytest_copilot-2.0.0.dist-info/WHEEL +4 -0
- pytest_copilot-2.0.0.dist-info/entry_points.txt +5 -0
- pytest_copilot-2.0.0.dist-info/licenses/LICENSE +21 -0
|
@@ -0,0 +1,133 @@
|
|
|
1
|
+
"""Configuration file loaders for Copilot and Claude Code projects.
|
|
2
|
+
|
|
3
|
+
Provides utilities for loading MCP server configurations from standard
|
|
4
|
+
config files (``.mcp.json``, ``.vscode/mcp.json``).
|
|
5
|
+
"""
|
|
6
|
+
|
|
7
|
+
from __future__ import annotations
|
|
8
|
+
|
|
9
|
+
from copy import deepcopy
|
|
10
|
+
from pathlib import Path
|
|
11
|
+
from typing import TYPE_CHECKING, Any
|
|
12
|
+
|
|
13
|
+
from pytest_copilot.core.plugin import _read_json
|
|
14
|
+
from pytest_copilot.core.validation import _validate_mcp_servers
|
|
15
|
+
|
|
16
|
+
if TYPE_CHECKING:
|
|
17
|
+
from pytest_copilot.copilot.contracts import CopilotTaskConfig
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
def snapshot_session_configuration(
|
|
21
|
+
agent: CopilotTaskConfig, session_config: dict[str, Any]
|
|
22
|
+
) -> dict[str, Any]:
|
|
23
|
+
"""Capture prepared settings without handlers, credentials, or server environments."""
|
|
24
|
+
system = session_config.get("system_message") or {}
|
|
25
|
+
snapshot = {
|
|
26
|
+
"name": agent.name,
|
|
27
|
+
"model": session_config.get("model"),
|
|
28
|
+
"instructions": system.get("content"),
|
|
29
|
+
"system_message_mode": system.get("mode"),
|
|
30
|
+
"reasoning_effort": session_config.get("reasoning_effort"),
|
|
31
|
+
"client_mode": agent.client_mode,
|
|
32
|
+
"image_detail": agent.image_detail,
|
|
33
|
+
"audit_requests": agent.audit_requests,
|
|
34
|
+
"max_tool_calls": agent.max_tool_calls,
|
|
35
|
+
"max_turns": agent.max_turns,
|
|
36
|
+
"timeout_s": agent.timeout_s,
|
|
37
|
+
"auto_confirm": agent.auto_confirm,
|
|
38
|
+
"allowed_tools": session_config.get("available_tools"),
|
|
39
|
+
"excluded_tools": session_config.get("excluded_tools"),
|
|
40
|
+
"skill_directories": session_config.get("skill_directories", []),
|
|
41
|
+
"disabled_skills": session_config.get("disabled_skills", []),
|
|
42
|
+
"active_agent": session_config.get("agent"),
|
|
43
|
+
"working_directory": session_config.get("working_directory"),
|
|
44
|
+
"persona": type(agent.persona).__name__,
|
|
45
|
+
"tools": [
|
|
46
|
+
{
|
|
47
|
+
"name": tool.name,
|
|
48
|
+
"description": tool.description,
|
|
49
|
+
"parameters": tool.parameters,
|
|
50
|
+
"metadata": tool.metadata,
|
|
51
|
+
}
|
|
52
|
+
for tool in session_config.get("tools") or []
|
|
53
|
+
],
|
|
54
|
+
"mcp_servers": {
|
|
55
|
+
name: {"type": config.get("type"), "tools": config.get("tools")}
|
|
56
|
+
for name, config in (session_config.get("mcp_servers") or {}).items()
|
|
57
|
+
},
|
|
58
|
+
"custom_agents": [
|
|
59
|
+
{
|
|
60
|
+
key: value
|
|
61
|
+
for key, value in custom.items()
|
|
62
|
+
if key
|
|
63
|
+
in {
|
|
64
|
+
"name",
|
|
65
|
+
"prompt",
|
|
66
|
+
"description",
|
|
67
|
+
"display_name",
|
|
68
|
+
"model",
|
|
69
|
+
"reasoning_effort",
|
|
70
|
+
"tools",
|
|
71
|
+
"infer",
|
|
72
|
+
"skills",
|
|
73
|
+
}
|
|
74
|
+
}
|
|
75
|
+
for custom in session_config.get("custom_agents") or []
|
|
76
|
+
],
|
|
77
|
+
}
|
|
78
|
+
for key in (
|
|
79
|
+
"enable_config_discovery",
|
|
80
|
+
"enable_file_hooks",
|
|
81
|
+
"enable_on_demand_instruction_discovery",
|
|
82
|
+
"skip_custom_instructions",
|
|
83
|
+
"enable_managed_settings",
|
|
84
|
+
"enable_skills",
|
|
85
|
+
):
|
|
86
|
+
if key in session_config:
|
|
87
|
+
snapshot[key] = session_config[key]
|
|
88
|
+
return deepcopy(snapshot)
|
|
89
|
+
|
|
90
|
+
|
|
91
|
+
def load_mcp_config(path: str | Path) -> dict[str, dict[str, Any]]:
|
|
92
|
+
"""Load MCP server configs from a ``.mcp.json`` file.
|
|
93
|
+
|
|
94
|
+
Supports both Claude Code / root-level ``.mcp.json`` and VS Code
|
|
95
|
+
``.vscode/mcp.json`` formats. Handles two common top-level keys:
|
|
96
|
+
|
|
97
|
+
* ``mcpServers`` — Claude Code / standard MCP convention
|
|
98
|
+
* ``servers`` — VS Code ``mcp.json`` convention
|
|
99
|
+
|
|
100
|
+
Exactly one supported key must be present. An explicit empty mapping is valid;
|
|
101
|
+
missing keys, conflicting keys, and malformed server definitions raise errors.
|
|
102
|
+
|
|
103
|
+
Returns:
|
|
104
|
+
Dict of ``server_name → config`` compatible with
|
|
105
|
+
``CopilotTask(mcp_servers=...)``.
|
|
106
|
+
|
|
107
|
+
Raises:
|
|
108
|
+
FileNotFoundError: If the file does not exist.
|
|
109
|
+
ValueError: If the file is not valid JSON or has unexpected structure.
|
|
110
|
+
|
|
111
|
+
Example::
|
|
112
|
+
|
|
113
|
+
from pytest_copilot.copilot.config import load_mcp_config
|
|
114
|
+
|
|
115
|
+
servers = load_mcp_config(".mcp.json")
|
|
116
|
+
agent = CopilotTask(mcp_servers=servers)
|
|
117
|
+
"""
|
|
118
|
+
path = Path(path)
|
|
119
|
+
if not path.is_file():
|
|
120
|
+
msg = f"MCP config file not found: {path}"
|
|
121
|
+
raise FileNotFoundError(msg)
|
|
122
|
+
|
|
123
|
+
raw = _read_json(path)
|
|
124
|
+
|
|
125
|
+
if not isinstance(raw, dict):
|
|
126
|
+
msg = f"{path}: MCP config must be a JSON object, got {type(raw).__name__}"
|
|
127
|
+
raise ValueError(msg)
|
|
128
|
+
|
|
129
|
+
fields = [key for key in ("mcpServers", "servers") if key in raw]
|
|
130
|
+
if len(fields) != 1:
|
|
131
|
+
raise ValueError(f"{path}: specify exactly one of 'mcpServers' and 'servers'")
|
|
132
|
+
|
|
133
|
+
return _validate_mcp_servers(raw[fields[0]], path, field=fields[0])
|
|
@@ -0,0 +1,222 @@
|
|
|
1
|
+
"""Shared Copilot runtime contracts and SDK-backed type aliases."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from collections.abc import Awaitable, Callable, Mapping
|
|
6
|
+
from typing import Any, Literal, Protocol, TypeAlias
|
|
7
|
+
|
|
8
|
+
from copilot.session import CustomAgentConfig, MCPServerConfig, ReasoningEffort, SessionHooks
|
|
9
|
+
|
|
10
|
+
from pytest_copilot.copilot.result import CopilotResult, CopilotResultAgent
|
|
11
|
+
|
|
12
|
+
CopilotCustomAgentConfig: TypeAlias = CustomAgentConfig
|
|
13
|
+
CopilotMCPServerConfig: TypeAlias = MCPServerConfig
|
|
14
|
+
CopilotReasoningEffort: TypeAlias = ReasoningEffort
|
|
15
|
+
CopilotSessionHooks: TypeAlias = SessionHooks
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
class CopilotRunResult(Protocol):
|
|
19
|
+
"""Result surface needed by recursive Copilot execution."""
|
|
20
|
+
|
|
21
|
+
agent: CopilotResultAgent | None
|
|
22
|
+
|
|
23
|
+
@property
|
|
24
|
+
def success(self) -> bool:
|
|
25
|
+
raise NotImplementedError
|
|
26
|
+
|
|
27
|
+
@property
|
|
28
|
+
def error(self) -> str | None:
|
|
29
|
+
raise NotImplementedError
|
|
30
|
+
|
|
31
|
+
@property
|
|
32
|
+
def final_response(self) -> str | None:
|
|
33
|
+
raise NotImplementedError
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
class CopilotEventMapper(Protocol):
|
|
37
|
+
"""Subagent lifecycle operations needed by persona dispatch tools."""
|
|
38
|
+
|
|
39
|
+
def record_subagent_start(self, *, invocation_id: str, name: str) -> None:
|
|
40
|
+
raise NotImplementedError
|
|
41
|
+
|
|
42
|
+
def record_subagent_complete(self, *, invocation_id: str, name: str) -> None:
|
|
43
|
+
raise NotImplementedError
|
|
44
|
+
|
|
45
|
+
def record_subagent_failed(self, *, invocation_id: str, name: str) -> None:
|
|
46
|
+
raise NotImplementedError
|
|
47
|
+
|
|
48
|
+
def record_subagent_result(
|
|
49
|
+
self, *, invocation_id: str, name: str, result: CopilotResult
|
|
50
|
+
) -> None:
|
|
51
|
+
raise NotImplementedError
|
|
52
|
+
|
|
53
|
+
|
|
54
|
+
class CopilotTaskConfig(CopilotResultAgent, Protocol):
|
|
55
|
+
"""Configuration surface consumed by the runner and personas."""
|
|
56
|
+
|
|
57
|
+
@property
|
|
58
|
+
def client_mode(self) -> Literal["copilot-cli", "empty"]:
|
|
59
|
+
raise NotImplementedError
|
|
60
|
+
|
|
61
|
+
@property
|
|
62
|
+
def image_detail(self) -> Literal["auto", "low", "high"] | None:
|
|
63
|
+
raise NotImplementedError
|
|
64
|
+
|
|
65
|
+
@property
|
|
66
|
+
def audit_requests(self) -> bool:
|
|
67
|
+
raise NotImplementedError
|
|
68
|
+
|
|
69
|
+
@property
|
|
70
|
+
def max_tool_calls(self) -> int | None:
|
|
71
|
+
raise NotImplementedError
|
|
72
|
+
|
|
73
|
+
@property
|
|
74
|
+
def name(self) -> str:
|
|
75
|
+
raise NotImplementedError
|
|
76
|
+
|
|
77
|
+
@property
|
|
78
|
+
def model(self) -> str | None:
|
|
79
|
+
raise NotImplementedError
|
|
80
|
+
|
|
81
|
+
@property
|
|
82
|
+
def reasoning_effort(self) -> CopilotReasoningEffort | None:
|
|
83
|
+
raise NotImplementedError
|
|
84
|
+
|
|
85
|
+
@property
|
|
86
|
+
def allowed_tools(self) -> list[str] | None:
|
|
87
|
+
raise NotImplementedError
|
|
88
|
+
|
|
89
|
+
@property
|
|
90
|
+
def excluded_tools(self) -> list[str] | None:
|
|
91
|
+
raise NotImplementedError
|
|
92
|
+
|
|
93
|
+
@property
|
|
94
|
+
def max_turns(self) -> int:
|
|
95
|
+
raise NotImplementedError
|
|
96
|
+
|
|
97
|
+
@property
|
|
98
|
+
def timeout_s(self) -> float:
|
|
99
|
+
raise NotImplementedError
|
|
100
|
+
|
|
101
|
+
@property
|
|
102
|
+
def auto_confirm(self) -> bool:
|
|
103
|
+
raise NotImplementedError
|
|
104
|
+
|
|
105
|
+
@property
|
|
106
|
+
def mcp_servers(self) -> dict[str, CopilotMCPServerConfig]:
|
|
107
|
+
raise NotImplementedError
|
|
108
|
+
|
|
109
|
+
@property
|
|
110
|
+
def custom_agents(self) -> list[CopilotCustomAgentConfig]:
|
|
111
|
+
raise NotImplementedError
|
|
112
|
+
|
|
113
|
+
@property
|
|
114
|
+
def skill_directories(self) -> list[str]:
|
|
115
|
+
raise NotImplementedError
|
|
116
|
+
|
|
117
|
+
@property
|
|
118
|
+
def persona(self) -> CopilotPersona:
|
|
119
|
+
raise NotImplementedError
|
|
120
|
+
|
|
121
|
+
def build_session_config(self) -> dict[str, Any]:
|
|
122
|
+
raise NotImplementedError
|
|
123
|
+
|
|
124
|
+
def create_subagent(
|
|
125
|
+
self,
|
|
126
|
+
*,
|
|
127
|
+
name: str,
|
|
128
|
+
model: str | None,
|
|
129
|
+
reasoning_effort: CopilotReasoningEffort | None,
|
|
130
|
+
instructions: str,
|
|
131
|
+
timeout_s: float,
|
|
132
|
+
max_turns: int,
|
|
133
|
+
allowed_tools: list[str] | None,
|
|
134
|
+
mcp_servers: dict[str, CopilotMCPServerConfig],
|
|
135
|
+
) -> CopilotTaskConfig:
|
|
136
|
+
raise NotImplementedError
|
|
137
|
+
|
|
138
|
+
|
|
139
|
+
CopilotNestedRunner: TypeAlias = Callable[
|
|
140
|
+
[CopilotTaskConfig, str],
|
|
141
|
+
Awaitable["CopilotResult"],
|
|
142
|
+
]
|
|
143
|
+
|
|
144
|
+
|
|
145
|
+
class CopilotPersona(Protocol):
|
|
146
|
+
"""Persona behavior required by a Copilot eval configuration."""
|
|
147
|
+
|
|
148
|
+
def apply(
|
|
149
|
+
self,
|
|
150
|
+
agent: CopilotTaskConfig,
|
|
151
|
+
session_config: dict[str, Any],
|
|
152
|
+
mapper: CopilotEventMapper,
|
|
153
|
+
nested_runner: CopilotNestedRunner,
|
|
154
|
+
) -> None:
|
|
155
|
+
raise NotImplementedError
|
|
156
|
+
|
|
157
|
+
|
|
158
|
+
def declared_agent_tools(
|
|
159
|
+
agent_config: CopilotCustomAgentConfig,
|
|
160
|
+
) -> list[str] | None:
|
|
161
|
+
"""Return the exact tool allowlist declared on a custom agent."""
|
|
162
|
+
if "tools" not in agent_config:
|
|
163
|
+
return None
|
|
164
|
+
return agent_config["tools"]
|
|
165
|
+
|
|
166
|
+
|
|
167
|
+
def merge_agent_mcp_servers(
|
|
168
|
+
parent_servers: Mapping[str, CopilotMCPServerConfig],
|
|
169
|
+
agent_config: CopilotCustomAgentConfig,
|
|
170
|
+
) -> dict[str, CopilotMCPServerConfig]:
|
|
171
|
+
"""Merge parent MCP servers with any exact custom-agent overrides."""
|
|
172
|
+
merged = dict(parent_servers)
|
|
173
|
+
agent_servers = agent_config.get("mcp_servers")
|
|
174
|
+
if agent_servers:
|
|
175
|
+
merged.update(agent_servers)
|
|
176
|
+
return merged
|
|
177
|
+
|
|
178
|
+
|
|
179
|
+
def resolve_agent_reasoning_effort(
|
|
180
|
+
*,
|
|
181
|
+
parent_model: str | None,
|
|
182
|
+
parent_reasoning_effort: CopilotReasoningEffort | None,
|
|
183
|
+
agent_config: CopilotCustomAgentConfig,
|
|
184
|
+
) -> CopilotReasoningEffort | None:
|
|
185
|
+
"""Resolve a custom agent's reasoning effort like the current SDK contract."""
|
|
186
|
+
if "reasoning_effort" in agent_config:
|
|
187
|
+
return agent_config["reasoning_effort"]
|
|
188
|
+
|
|
189
|
+
agent_model = agent_config.get("model")
|
|
190
|
+
if agent_model is None or agent_model == parent_model:
|
|
191
|
+
return parent_reasoning_effort
|
|
192
|
+
|
|
193
|
+
return None
|
|
194
|
+
|
|
195
|
+
|
|
196
|
+
def require_custom_agent_prompt(
|
|
197
|
+
agent_config: CopilotCustomAgentConfig,
|
|
198
|
+
) -> str:
|
|
199
|
+
"""Return the declared prompt or raise when the runtime contract is broken."""
|
|
200
|
+
prompt = agent_config.get("prompt")
|
|
201
|
+
if isinstance(prompt, str) and prompt:
|
|
202
|
+
return prompt
|
|
203
|
+
msg = f"Custom agent '{agent_config.get('name', 'unknown')}' is missing a prompt"
|
|
204
|
+
raise ValueError(msg)
|
|
205
|
+
|
|
206
|
+
|
|
207
|
+
def require_custom_agent_name(
|
|
208
|
+
agent_config: CopilotCustomAgentConfig,
|
|
209
|
+
) -> str:
|
|
210
|
+
"""Return the declared machine name or raise when the runtime contract is broken."""
|
|
211
|
+
name = agent_config.get("name")
|
|
212
|
+
if isinstance(name, str) and name:
|
|
213
|
+
return name
|
|
214
|
+
raise ValueError("Custom agent is missing a name")
|
|
215
|
+
|
|
216
|
+
|
|
217
|
+
def require_mapping(arguments: Any) -> dict[str, Any]:
|
|
218
|
+
"""Validate tool arguments that must be object-shaped."""
|
|
219
|
+
if isinstance(arguments, dict):
|
|
220
|
+
return arguments
|
|
221
|
+
msg = "Custom-agent dispatch arguments must be an object"
|
|
222
|
+
raise ValueError(msg)
|
|
@@ -0,0 +1,230 @@
|
|
|
1
|
+
"""Per-run dispatch limits and ownership of custom tool work."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import asyncio
|
|
6
|
+
import inspect
|
|
7
|
+
import logging
|
|
8
|
+
from collections.abc import Awaitable
|
|
9
|
+
from dataclasses import replace
|
|
10
|
+
from typing import Any, TypeVar, cast
|
|
11
|
+
|
|
12
|
+
from copilot.session import PreToolUseHookInput, PreToolUseHookOutput
|
|
13
|
+
from copilot.tools import Tool, ToolInvocation, ToolResult
|
|
14
|
+
|
|
15
|
+
T = TypeVar("T")
|
|
16
|
+
logger = logging.getLogger(__name__)
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
async def resolve(value: T | Awaitable[T]) -> T:
|
|
20
|
+
return await value if inspect.isawaitable(value) else cast(T, value)
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
class RunControls:
|
|
24
|
+
"""Close dispatch before aborting, and drain owned work before returning."""
|
|
25
|
+
|
|
26
|
+
def __init__(self, max_tool_calls: int | None) -> None:
|
|
27
|
+
self.limit = max_tool_calls
|
|
28
|
+
self.admitted = 0
|
|
29
|
+
self.completed = 0
|
|
30
|
+
self.started = 0
|
|
31
|
+
self.handlers_started = 0
|
|
32
|
+
self.handlers_finished = 0
|
|
33
|
+
self.closed = False
|
|
34
|
+
self._budget_exhausted = False
|
|
35
|
+
self._started_call_ids: list[str] = []
|
|
36
|
+
self._completed_call_ids: set[str] = set()
|
|
37
|
+
self._handler_started_call_ids: set[str] = set()
|
|
38
|
+
self._handler_finished_call_ids: set[str] = set()
|
|
39
|
+
self.budget_exceeded = asyncio.Event()
|
|
40
|
+
self.failed = asyncio.Event()
|
|
41
|
+
self.error: str | None = None
|
|
42
|
+
self.lock = asyncio.Lock()
|
|
43
|
+
self.active: set[asyncio.Task[ToolResult]] = set()
|
|
44
|
+
|
|
45
|
+
def install(self, config: dict[str, Any], caller_hooks: dict[str, Any]) -> None:
|
|
46
|
+
hooks = dict(config.get("hooks") or {})
|
|
47
|
+
pre_hooks = []
|
|
48
|
+
for source in (caller_hooks, hooks):
|
|
49
|
+
pre = source.get("on_pre_tool_use")
|
|
50
|
+
if pre is not None and pre not in pre_hooks:
|
|
51
|
+
pre_hooks.append(pre)
|
|
52
|
+
for key, handler in caller_hooks.items():
|
|
53
|
+
if key == "on_pre_tool_use":
|
|
54
|
+
continue
|
|
55
|
+
if key in hooks and hooks[key] is not handler:
|
|
56
|
+
raise ValueError(f"Persona replaced caller hook {key}; compose it explicitly")
|
|
57
|
+
hooks[key] = handler
|
|
58
|
+
|
|
59
|
+
async def before(
|
|
60
|
+
data: PreToolUseHookInput, context: dict[str, str]
|
|
61
|
+
) -> PreToolUseHookOutput | None:
|
|
62
|
+
async with self.lock:
|
|
63
|
+
if self.closed:
|
|
64
|
+
return {
|
|
65
|
+
"permissionDecision": "deny",
|
|
66
|
+
"permissionDecisionReason": "Eval stopped",
|
|
67
|
+
}
|
|
68
|
+
if self.limit is not None and self.admitted >= self.limit:
|
|
69
|
+
self.closed = True
|
|
70
|
+
self._budget_exhausted = True
|
|
71
|
+
logger.info(
|
|
72
|
+
"Tool budget closed admission: admitted=%d started=%d completed=%d",
|
|
73
|
+
self.admitted,
|
|
74
|
+
self.started,
|
|
75
|
+
self.completed,
|
|
76
|
+
)
|
|
77
|
+
self._signal_budget_when_complete()
|
|
78
|
+
return {
|
|
79
|
+
"permissionDecision": "deny",
|
|
80
|
+
"permissionDecisionReason": "Eval tool-call budget exhausted",
|
|
81
|
+
}
|
|
82
|
+
combined: PreToolUseHookOutput = {}
|
|
83
|
+
for hook in pre_hooks:
|
|
84
|
+
try:
|
|
85
|
+
output = await resolve(hook(data, context))
|
|
86
|
+
if output is not None and (
|
|
87
|
+
not isinstance(output, dict)
|
|
88
|
+
or output.get("permissionDecision")
|
|
89
|
+
not in (None, "allow", "deny", "ask")
|
|
90
|
+
):
|
|
91
|
+
raise ValueError("Invalid pre-tool guard output")
|
|
92
|
+
except Exception as exc:
|
|
93
|
+
self.error = f"Pre-tool guard failed: {type(exc).__name__}"
|
|
94
|
+
self.closed = True
|
|
95
|
+
self.failed.set()
|
|
96
|
+
return {
|
|
97
|
+
"permissionDecision": "deny",
|
|
98
|
+
"permissionDecisionReason": self.error,
|
|
99
|
+
}
|
|
100
|
+
if output:
|
|
101
|
+
must_ask = combined.get("permissionDecision") == "ask"
|
|
102
|
+
previous_context = combined.get("additionalContext")
|
|
103
|
+
combined.update(output)
|
|
104
|
+
if output.get("permissionDecision") == "deny":
|
|
105
|
+
return combined
|
|
106
|
+
if must_ask:
|
|
107
|
+
combined["permissionDecision"] = "ask"
|
|
108
|
+
if previous_context and output.get("additionalContext"):
|
|
109
|
+
combined["additionalContext"] = (
|
|
110
|
+
previous_context + "\n" + output["additionalContext"]
|
|
111
|
+
)
|
|
112
|
+
if "modifiedArgs" in output:
|
|
113
|
+
data = {**data, "toolArgs": output["modifiedArgs"]}
|
|
114
|
+
# A timeout may have closed dispatch while a caller hook awaited.
|
|
115
|
+
if self.closed:
|
|
116
|
+
return {
|
|
117
|
+
"permissionDecision": "deny",
|
|
118
|
+
"permissionDecisionReason": "Eval stopped",
|
|
119
|
+
}
|
|
120
|
+
self.admitted += 1
|
|
121
|
+
return combined or None
|
|
122
|
+
|
|
123
|
+
hooks["on_pre_tool_use"] = before
|
|
124
|
+
config["hooks"] = hooks
|
|
125
|
+
config["tools"] = [self.wrap(tool) for tool in config.get("tools", [])]
|
|
126
|
+
|
|
127
|
+
def observe(self, event: Any) -> bool:
|
|
128
|
+
"""Observe tool lifecycle and report when the budget barrier releases."""
|
|
129
|
+
event_type = event.type.value if hasattr(event.type, "value") else str(event.type)
|
|
130
|
+
if event_type == "tool.execution_start":
|
|
131
|
+
call_id = _event_data_field(event, "tool_call_id", "")
|
|
132
|
+
if call_id and call_id not in self._started_call_ids:
|
|
133
|
+
self._started_call_ids.append(call_id)
|
|
134
|
+
self.started += 1
|
|
135
|
+
return False
|
|
136
|
+
if event_type != "tool.execution_complete":
|
|
137
|
+
return False
|
|
138
|
+
call_id = _event_data_field(event, "tool_call_id", "")
|
|
139
|
+
if call_id and call_id not in self._completed_call_ids:
|
|
140
|
+
self._completed_call_ids.add(call_id)
|
|
141
|
+
self.completed += 1
|
|
142
|
+
return self._signal_budget_when_complete()
|
|
143
|
+
|
|
144
|
+
def _signal_budget_when_complete(self) -> bool:
|
|
145
|
+
if (
|
|
146
|
+
self._budget_exhausted
|
|
147
|
+
and self.started >= self.admitted
|
|
148
|
+
and set(self._started_call_ids) <= self._completed_call_ids
|
|
149
|
+
and not self.budget_exceeded.is_set()
|
|
150
|
+
):
|
|
151
|
+
logger.info(
|
|
152
|
+
"Tool budget completion barrier released: admitted=%d started=%d completed=%d",
|
|
153
|
+
self.admitted,
|
|
154
|
+
self.started,
|
|
155
|
+
self.completed,
|
|
156
|
+
)
|
|
157
|
+
self.budget_exceeded.set()
|
|
158
|
+
return True
|
|
159
|
+
return False
|
|
160
|
+
|
|
161
|
+
def wrap(self, tool: Tool) -> Tool:
|
|
162
|
+
handler = tool.handler
|
|
163
|
+
if handler is None:
|
|
164
|
+
return tool
|
|
165
|
+
|
|
166
|
+
async def invoke(invocation: ToolInvocation) -> ToolResult:
|
|
167
|
+
async def execute() -> ToolResult:
|
|
168
|
+
call_id = invocation.tool_call_id
|
|
169
|
+
self._handler_started_call_ids.add(call_id)
|
|
170
|
+
self.handlers_started += 1
|
|
171
|
+
try:
|
|
172
|
+
return await resolve(handler(invocation))
|
|
173
|
+
finally:
|
|
174
|
+
self._handler_finished_call_ids.add(call_id)
|
|
175
|
+
self.handlers_finished += 1
|
|
176
|
+
|
|
177
|
+
task = asyncio.create_task(execute())
|
|
178
|
+
self.active.add(task)
|
|
179
|
+
try:
|
|
180
|
+
# SDK abort/disconnect may cancel the RPC callback, not the action.
|
|
181
|
+
return await asyncio.shield(task)
|
|
182
|
+
finally:
|
|
183
|
+
if task.done():
|
|
184
|
+
self.active.discard(task)
|
|
185
|
+
|
|
186
|
+
return replace(tool, handler=invoke)
|
|
187
|
+
|
|
188
|
+
def incomplete_diagnostics(self, *, abort_phase: str) -> list[str]:
|
|
189
|
+
"""Return safe lifecycle details for started calls missing SDK completion."""
|
|
190
|
+
diagnostics = []
|
|
191
|
+
for ordinal, call_id in enumerate(self._started_call_ids, 1):
|
|
192
|
+
if call_id in self._completed_call_ids:
|
|
193
|
+
continue
|
|
194
|
+
handler_state = (
|
|
195
|
+
"finished"
|
|
196
|
+
if call_id in self._handler_finished_call_ids
|
|
197
|
+
else "started"
|
|
198
|
+
if call_id in self._handler_started_call_ids
|
|
199
|
+
else "not_started"
|
|
200
|
+
)
|
|
201
|
+
diagnostics.append(
|
|
202
|
+
"Tool lifecycle incomplete: "
|
|
203
|
+
f"call={call_id}, start_ordinal={ordinal}, admitted={self.admitted}, "
|
|
204
|
+
f"handler={handler_state}, sdk_completion=missing, abort_phase={abort_phase}"
|
|
205
|
+
)
|
|
206
|
+
return diagnostics
|
|
207
|
+
|
|
208
|
+
async def drain(self) -> list[str]:
|
|
209
|
+
"""Never release a trial while its local handlers can still act."""
|
|
210
|
+
self.closed = True
|
|
211
|
+
tasks = list(self.active)
|
|
212
|
+
if not tasks:
|
|
213
|
+
return []
|
|
214
|
+
outcomes = await asyncio.shield(asyncio.gather(*tasks, return_exceptions=True))
|
|
215
|
+
self.active.difference_update(tasks)
|
|
216
|
+
return [
|
|
217
|
+
f"Tool handler failed during cleanup: {type(outcome).__name__}"
|
|
218
|
+
for outcome in outcomes
|
|
219
|
+
if isinstance(outcome, BaseException)
|
|
220
|
+
]
|
|
221
|
+
|
|
222
|
+
|
|
223
|
+
def _event_data_field(event: Any, name: str, default: T) -> T:
|
|
224
|
+
data = getattr(event, "data", None)
|
|
225
|
+
if isinstance(data, dict):
|
|
226
|
+
camel_name = "".join(
|
|
227
|
+
part if index == 0 else part.title() for index, part in enumerate(name.split("_"))
|
|
228
|
+
)
|
|
229
|
+
return cast(T, data.get(name, data.get(camel_name, default)))
|
|
230
|
+
return cast(T, getattr(data, name, default))
|