pytest-copilot 2.0.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- pytest_copilot/__init__.py +133 -0
- pytest_copilot/copilot/__init__.py +47 -0
- pytest_copilot/copilot/_command_job.py +109 -0
- pytest_copilot/copilot/api.py +12 -0
- pytest_copilot/copilot/client.py +58 -0
- pytest_copilot/copilot/commands.py +265 -0
- pytest_copilot/copilot/config.py +133 -0
- pytest_copilot/copilot/contracts.py +222 -0
- pytest_copilot/copilot/controls.py +230 -0
- pytest_copilot/copilot/events.py +795 -0
- pytest_copilot/copilot/fixtures.py +289 -0
- pytest_copilot/copilot/personas.py +607 -0
- pytest_copilot/copilot/requests.py +350 -0
- pytest_copilot/copilot/result.py +309 -0
- pytest_copilot/copilot/runner.py +266 -0
- pytest_copilot/copilot/skills.py +104 -0
- pytest_copilot/copilot/task.py +546 -0
- pytest_copilot/core/__init__.py +77 -0
- pytest_copilot/core/errors.py +44 -0
- pytest_copilot/core/evals.py +521 -0
- pytest_copilot/core/plugin.py +403 -0
- pytest_copilot/core/prompt.py +167 -0
- pytest_copilot/core/result.py +543 -0
- pytest_copilot/core/serialization.py +128 -0
- pytest_copilot/core/skill.py +406 -0
- pytest_copilot/core/skill_evals.py +148 -0
- pytest_copilot/core/skill_grading.py +78 -0
- pytest_copilot/core/validation.py +83 -0
- pytest_copilot/execution/__init__.py +15 -0
- pytest_copilot/execution/cost.py +159 -0
- pytest_copilot/execution/rate_limiter.py +177 -0
- pytest_copilot/execution/servers.py +442 -0
- pytest_copilot/execution/skill_tools.py +228 -0
- pytest_copilot/fixtures/__init__.py +18 -0
- pytest_copilot/fixtures/factories.py +44 -0
- pytest_copilot/fixtures/iteration.py +16 -0
- pytest_copilot/onboarding.py +413 -0
- pytest_copilot/plugin.py +562 -0
- pytest_copilot/plugin_options.py +32 -0
- pytest_copilot/plugin_report.py +18 -0
- pytest_copilot/reporting/__init__.py +16 -0
- pytest_copilot/reporting/collector.py +203 -0
- pytest_copilot/reporting/generator.py +55 -0
- pytest_copilot/reporting/schema.py +5 -0
- pytest_copilot/testing/__init__.py +17 -0
- pytest_copilot/testing/banking.py +479 -0
- pytest_copilot/testing/banking_mcp.py +167 -0
- pytest_copilot/testing/todo.py +380 -0
- pytest_copilot/testing/todo_mcp.py +156 -0
- pytest_copilot/testing/types.py +31 -0
- pytest_copilot-2.0.0.dist-info/METADATA +177 -0
- pytest_copilot-2.0.0.dist-info/RECORD +55 -0
- pytest_copilot-2.0.0.dist-info/WHEEL +4 -0
- pytest_copilot-2.0.0.dist-info/entry_points.txt +5 -0
- pytest_copilot-2.0.0.dist-info/licenses/LICENSE +21 -0
|
@@ -0,0 +1,133 @@
|
|
|
1
|
+
"""Copilot task execution and evidence for testing AI-facing tools and skills."""
|
|
2
|
+
|
|
3
|
+
import logging
|
|
4
|
+
|
|
5
|
+
# Configure library logging per Python best practices:
|
|
6
|
+
# https://docs.python.org/3/howto/logging.html#configuring-logging-for-a-library
|
|
7
|
+
# Libraries should add NullHandler to prevent "No handler found" warnings
|
|
8
|
+
# when the application hasn't configured logging.
|
|
9
|
+
logging.getLogger(__name__).addHandler(logging.NullHandler())
|
|
10
|
+
|
|
11
|
+
# Core types # noqa: E402 - logging must be configured before submodule imports
|
|
12
|
+
# Copilot coding agent support (now required)
|
|
13
|
+
from pytest_copilot.copilot import ( # noqa: E402
|
|
14
|
+
ClaudeCodePersona,
|
|
15
|
+
CopilotCLIPersona,
|
|
16
|
+
CopilotResult,
|
|
17
|
+
CopilotTask,
|
|
18
|
+
HeadlessPersona,
|
|
19
|
+
Persona,
|
|
20
|
+
VSCodePersona,
|
|
21
|
+
run_copilot,
|
|
22
|
+
)
|
|
23
|
+
from pytest_copilot.core import ( # noqa: E402
|
|
24
|
+
AITestError,
|
|
25
|
+
EngineTimeoutError,
|
|
26
|
+
EvalResult,
|
|
27
|
+
HookDefinition,
|
|
28
|
+
ImageContent,
|
|
29
|
+
MCPPrompt,
|
|
30
|
+
MCPPromptArgument,
|
|
31
|
+
Plugin,
|
|
32
|
+
PluginMetadata,
|
|
33
|
+
Prompt,
|
|
34
|
+
ServerStartError,
|
|
35
|
+
Skill,
|
|
36
|
+
SkillError,
|
|
37
|
+
SkillEvalCase,
|
|
38
|
+
SkillInfo,
|
|
39
|
+
SkillMetadata,
|
|
40
|
+
SubagentInvocation,
|
|
41
|
+
ToolCall,
|
|
42
|
+
ToolInfo,
|
|
43
|
+
Turn,
|
|
44
|
+
export_grading,
|
|
45
|
+
has_skill_evals,
|
|
46
|
+
load_custom_agent,
|
|
47
|
+
load_custom_agents,
|
|
48
|
+
load_instruction_file,
|
|
49
|
+
load_instruction_files,
|
|
50
|
+
load_plugin,
|
|
51
|
+
load_prompt,
|
|
52
|
+
load_prompt_file,
|
|
53
|
+
load_prompt_files,
|
|
54
|
+
load_prompts,
|
|
55
|
+
load_skill,
|
|
56
|
+
load_skill_evals,
|
|
57
|
+
load_system_prompts,
|
|
58
|
+
)
|
|
59
|
+
from pytest_copilot.execution import ( # noqa: E402
|
|
60
|
+
MCPServer,
|
|
61
|
+
Wait,
|
|
62
|
+
WaitStrategy,
|
|
63
|
+
)
|
|
64
|
+
|
|
65
|
+
# Reporting
|
|
66
|
+
from pytest_copilot.reporting import ( # noqa: E402
|
|
67
|
+
SuiteReport,
|
|
68
|
+
TestReport,
|
|
69
|
+
build_suite_report,
|
|
70
|
+
generate_json,
|
|
71
|
+
load_suite_report,
|
|
72
|
+
)
|
|
73
|
+
|
|
74
|
+
__all__ = [ # noqa: RUF022
|
|
75
|
+
# Core
|
|
76
|
+
"AITestError",
|
|
77
|
+
"EngineTimeoutError",
|
|
78
|
+
"EvalResult",
|
|
79
|
+
"HookDefinition",
|
|
80
|
+
"ImageContent",
|
|
81
|
+
"MCPPrompt",
|
|
82
|
+
"MCPPromptArgument",
|
|
83
|
+
"Plugin",
|
|
84
|
+
"PluginMetadata",
|
|
85
|
+
"Prompt",
|
|
86
|
+
"ServerStartError",
|
|
87
|
+
"Skill",
|
|
88
|
+
"SkillError",
|
|
89
|
+
"SkillEvalCase",
|
|
90
|
+
"SkillInfo",
|
|
91
|
+
"SkillMetadata",
|
|
92
|
+
"SubagentInvocation",
|
|
93
|
+
"ToolCall",
|
|
94
|
+
"ToolInfo",
|
|
95
|
+
"Turn",
|
|
96
|
+
"export_grading",
|
|
97
|
+
"has_skill_evals",
|
|
98
|
+
"load_custom_agent",
|
|
99
|
+
"load_custom_agents",
|
|
100
|
+
"load_instruction_file",
|
|
101
|
+
"load_instruction_files",
|
|
102
|
+
"load_plugin",
|
|
103
|
+
"load_prompt",
|
|
104
|
+
"load_prompt_file",
|
|
105
|
+
"load_prompt_files",
|
|
106
|
+
"load_prompts",
|
|
107
|
+
"load_skill",
|
|
108
|
+
"load_skill_evals",
|
|
109
|
+
"load_system_prompts",
|
|
110
|
+
# Copilot (primary eval harness)
|
|
111
|
+
"ClaudeCodePersona",
|
|
112
|
+
"CopilotCLIPersona",
|
|
113
|
+
"CopilotTask",
|
|
114
|
+
"CopilotResult",
|
|
115
|
+
"HeadlessPersona",
|
|
116
|
+
"Persona",
|
|
117
|
+
"VSCodePersona",
|
|
118
|
+
"run_copilot",
|
|
119
|
+
# Server config types
|
|
120
|
+
"MCPServer",
|
|
121
|
+
"Wait",
|
|
122
|
+
"WaitStrategy",
|
|
123
|
+
# Reporting
|
|
124
|
+
"SuiteReport",
|
|
125
|
+
"TestReport",
|
|
126
|
+
"build_suite_report",
|
|
127
|
+
"generate_json",
|
|
128
|
+
"load_suite_report",
|
|
129
|
+
]
|
|
130
|
+
|
|
131
|
+
from importlib.metadata import version as _get_version # noqa: E402
|
|
132
|
+
|
|
133
|
+
__version__ = _get_version("pytest-copilot")
|
|
@@ -0,0 +1,47 @@
|
|
|
1
|
+
"""GitHub Copilot coding agent support for pytest-copilot.
|
|
2
|
+
|
|
3
|
+
Provides ``CopilotTask``, ``CopilotResult``, ``run_copilot``, and IDE
|
|
4
|
+
personas for testing real coding agents via the GitHub Copilot SDK.
|
|
5
|
+
|
|
6
|
+
Install with: ``uv add pytest-copilot``
|
|
7
|
+
"""
|
|
8
|
+
|
|
9
|
+
from __future__ import annotations
|
|
10
|
+
|
|
11
|
+
from pytest_copilot.copilot.api import run_copilot
|
|
12
|
+
from pytest_copilot.copilot.commands import CommandCancelledError, create_command_tool
|
|
13
|
+
from pytest_copilot.copilot.config import load_mcp_config
|
|
14
|
+
from pytest_copilot.copilot.fixtures import copilot_run
|
|
15
|
+
from pytest_copilot.copilot.personas import (
|
|
16
|
+
ClaudeCodePersona,
|
|
17
|
+
CopilotCLIPersona,
|
|
18
|
+
HeadlessPersona,
|
|
19
|
+
Persona,
|
|
20
|
+
VSCodePersona,
|
|
21
|
+
)
|
|
22
|
+
from pytest_copilot.copilot.requests import RequestAudit
|
|
23
|
+
from pytest_copilot.copilot.result import CopilotResult, UsageInfo
|
|
24
|
+
from pytest_copilot.copilot.task import CopilotTask
|
|
25
|
+
from pytest_copilot.core.evals import load_custom_agent, load_custom_agents
|
|
26
|
+
from pytest_copilot.core.result import DiscoveredSkill, SkillDiscovery
|
|
27
|
+
|
|
28
|
+
__all__ = [
|
|
29
|
+
"CommandCancelledError",
|
|
30
|
+
"create_command_tool",
|
|
31
|
+
"CopilotTask",
|
|
32
|
+
"CopilotResult",
|
|
33
|
+
"DiscoveredSkill",
|
|
34
|
+
"RequestAudit",
|
|
35
|
+
"SkillDiscovery",
|
|
36
|
+
"UsageInfo",
|
|
37
|
+
"ClaudeCodePersona",
|
|
38
|
+
"CopilotCLIPersona",
|
|
39
|
+
"HeadlessPersona",
|
|
40
|
+
"Persona",
|
|
41
|
+
"VSCodePersona",
|
|
42
|
+
"copilot_run",
|
|
43
|
+
"load_custom_agent",
|
|
44
|
+
"load_custom_agents",
|
|
45
|
+
"load_mcp_config",
|
|
46
|
+
"run_copilot",
|
|
47
|
+
]
|
|
@@ -0,0 +1,109 @@
|
|
|
1
|
+
"""Windows process ownership for the direct-command tool."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import ctypes
|
|
6
|
+
import sys
|
|
7
|
+
from ctypes import wintypes
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
def _last_error() -> OSError:
|
|
11
|
+
if sys.platform != "win32":
|
|
12
|
+
raise RuntimeError("Windows process jobs require Windows")
|
|
13
|
+
return ctypes.WinError(ctypes.get_last_error())
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
class _BasicLimits(ctypes.Structure):
|
|
17
|
+
_fields_ = [
|
|
18
|
+
("process_time", ctypes.c_int64),
|
|
19
|
+
("job_time", ctypes.c_int64),
|
|
20
|
+
("flags", wintypes.DWORD),
|
|
21
|
+
("minimum_working_set", ctypes.c_size_t),
|
|
22
|
+
("maximum_working_set", ctypes.c_size_t),
|
|
23
|
+
("active_process_limit", wintypes.DWORD),
|
|
24
|
+
("affinity", ctypes.c_size_t),
|
|
25
|
+
("priority_class", wintypes.DWORD),
|
|
26
|
+
("scheduling_class", wintypes.DWORD),
|
|
27
|
+
]
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
class _IOCounters(ctypes.Structure):
|
|
31
|
+
_fields_ = [
|
|
32
|
+
(name, ctypes.c_uint64)
|
|
33
|
+
for name in (
|
|
34
|
+
"read_ops",
|
|
35
|
+
"write_ops",
|
|
36
|
+
"other_ops",
|
|
37
|
+
"read_bytes",
|
|
38
|
+
"write_bytes",
|
|
39
|
+
"other_bytes",
|
|
40
|
+
)
|
|
41
|
+
]
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
class _ExtendedLimits(ctypes.Structure):
|
|
45
|
+
_fields_ = [
|
|
46
|
+
("basic", _BasicLimits),
|
|
47
|
+
("io", _IOCounters),
|
|
48
|
+
("process_memory_limit", ctypes.c_size_t),
|
|
49
|
+
("job_memory_limit", ctypes.c_size_t),
|
|
50
|
+
("peak_process_memory", ctypes.c_size_t),
|
|
51
|
+
("peak_job_memory", ctypes.c_size_t),
|
|
52
|
+
]
|
|
53
|
+
|
|
54
|
+
|
|
55
|
+
class CommandJob:
|
|
56
|
+
"""Keep the command and its descendants in a kill-on-close Windows job."""
|
|
57
|
+
|
|
58
|
+
def __init__(self) -> None:
|
|
59
|
+
if sys.platform != "win32":
|
|
60
|
+
raise OSError("Windows process jobs require Windows")
|
|
61
|
+
self.api = ctypes.WinDLL("kernel32", use_last_error=True)
|
|
62
|
+
self.api.CreateJobObjectW.argtypes = [ctypes.c_void_p, wintypes.LPCWSTR]
|
|
63
|
+
self.api.CreateJobObjectW.restype = wintypes.HANDLE
|
|
64
|
+
self.api.SetInformationJobObject.argtypes = [
|
|
65
|
+
wintypes.HANDLE,
|
|
66
|
+
ctypes.c_int,
|
|
67
|
+
ctypes.c_void_p,
|
|
68
|
+
wintypes.DWORD,
|
|
69
|
+
]
|
|
70
|
+
self.api.SetInformationJobObject.restype = wintypes.BOOL
|
|
71
|
+
self.api.OpenProcess.argtypes = [wintypes.DWORD, wintypes.BOOL, wintypes.DWORD]
|
|
72
|
+
self.api.OpenProcess.restype = wintypes.HANDLE
|
|
73
|
+
self.api.AssignProcessToJobObject.argtypes = [wintypes.HANDLE, wintypes.HANDLE]
|
|
74
|
+
self.api.AssignProcessToJobObject.restype = wintypes.BOOL
|
|
75
|
+
self.api.CloseHandle.argtypes = [wintypes.HANDLE]
|
|
76
|
+
self.api.CloseHandle.restype = wintypes.BOOL
|
|
77
|
+
self.handle = self.api.CreateJobObjectW(None, None)
|
|
78
|
+
if not self.handle:
|
|
79
|
+
raise _last_error()
|
|
80
|
+
limits = _ExtendedLimits()
|
|
81
|
+
limits.basic.flags = 0x2000 # JOB_OBJECT_LIMIT_KILL_ON_JOB_CLOSE
|
|
82
|
+
if not self.api.SetInformationJobObject(
|
|
83
|
+
self.handle, 9, ctypes.byref(limits), ctypes.sizeof(limits)
|
|
84
|
+
):
|
|
85
|
+
error = _last_error()
|
|
86
|
+
self.close()
|
|
87
|
+
raise error
|
|
88
|
+
|
|
89
|
+
def assign(self, pid: int) -> None:
|
|
90
|
+
"""Assign a newly started process before waiting for its output."""
|
|
91
|
+
if sys.platform != "win32":
|
|
92
|
+
raise OSError("Windows process jobs require Windows")
|
|
93
|
+
handle = self.api.OpenProcess(0x0101, False, pid) # TERMINATE | SET_QUOTA
|
|
94
|
+
if not handle:
|
|
95
|
+
raise _last_error()
|
|
96
|
+
try:
|
|
97
|
+
if not self.api.AssignProcessToJobObject(self.handle, handle):
|
|
98
|
+
raise _last_error()
|
|
99
|
+
finally:
|
|
100
|
+
self.api.CloseHandle(handle)
|
|
101
|
+
|
|
102
|
+
def close(self) -> None:
|
|
103
|
+
"""Terminate all processes still owned by the job."""
|
|
104
|
+
if sys.platform != "win32":
|
|
105
|
+
raise OSError("Windows process jobs require Windows")
|
|
106
|
+
if self.handle:
|
|
107
|
+
if not self.api.CloseHandle(self.handle):
|
|
108
|
+
raise _last_error()
|
|
109
|
+
self.handle = None
|
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
"""Typed public entry points for Copilot execution."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from pytest_copilot.copilot.result import CopilotResult
|
|
6
|
+
from pytest_copilot.copilot.runner import run_copilot as _run_copilot
|
|
7
|
+
from pytest_copilot.copilot.task import CopilotTask
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
async def run_copilot(agent: CopilotTask, prompt: str) -> CopilotResult:
|
|
11
|
+
"""Execute a prompt and return the concrete public result type."""
|
|
12
|
+
return await _run_copilot(agent, prompt)
|
|
@@ -0,0 +1,58 @@
|
|
|
1
|
+
"""SDK client setup and cleanup for eval sessions."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import asyncio
|
|
6
|
+
import logging
|
|
7
|
+
import os
|
|
8
|
+
from typing import Any
|
|
9
|
+
|
|
10
|
+
from copilot import CopilotClientMode, CopilotRequestHandler
|
|
11
|
+
from copilot.client import CopilotClient
|
|
12
|
+
from copilot.generated.rpc import PermissionDecisionApproveOnce
|
|
13
|
+
|
|
14
|
+
logger = logging.getLogger(__name__)
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
def get_github_token() -> str | None:
|
|
18
|
+
"""Resolve explicit GitHub credentials without changing the environment."""
|
|
19
|
+
return os.environ.get("GITHUB_TOKEN") or os.environ.get("GH_TOKEN")
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
def create_client(
|
|
23
|
+
working_directory: str = ".",
|
|
24
|
+
*,
|
|
25
|
+
mode: CopilotClientMode = "copilot-cli",
|
|
26
|
+
base_directory: str | None = None,
|
|
27
|
+
request_handler: CopilotRequestHandler | None = None,
|
|
28
|
+
) -> CopilotClient:
|
|
29
|
+
"""Create a client using explicit credentials or the SDK's signed-in user."""
|
|
30
|
+
return CopilotClient(
|
|
31
|
+
working_directory=working_directory,
|
|
32
|
+
log_level="warning",
|
|
33
|
+
github_token=get_github_token(),
|
|
34
|
+
mode=mode,
|
|
35
|
+
base_directory=base_directory,
|
|
36
|
+
request_handler=request_handler,
|
|
37
|
+
)
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
def approve_all_permissions(*_args: Any, **_kwargs: Any) -> PermissionDecisionApproveOnce:
|
|
41
|
+
"""Approve a permission request when the caller enables automatic approval."""
|
|
42
|
+
return PermissionDecisionApproveOnce()
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
async def stop_client(client: CopilotClient) -> list[str]:
|
|
46
|
+
"""Release the owned runtime without masking an execution failure."""
|
|
47
|
+
try:
|
|
48
|
+
await asyncio.wait_for(client.stop(), timeout=30)
|
|
49
|
+
except Exception as exc:
|
|
50
|
+
logger.warning("Failed to stop Copilot CLI cleanly, force stopping", exc_info=True)
|
|
51
|
+
errors = [f"Runtime cleanup failed: {type(exc).__name__}"]
|
|
52
|
+
try:
|
|
53
|
+
await asyncio.wait_for(client.force_stop(), timeout=10)
|
|
54
|
+
except Exception as force_exc:
|
|
55
|
+
logger.error("Failed to force stop Copilot CLI", exc_info=True)
|
|
56
|
+
errors.append(f"Runtime force stop failed: {type(force_exc).__name__}")
|
|
57
|
+
return errors
|
|
58
|
+
return []
|
|
@@ -0,0 +1,265 @@
|
|
|
1
|
+
"""Direct executable tools: argument arrays, no shell and no persistent server."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import asyncio
|
|
6
|
+
import base64
|
|
7
|
+
import json
|
|
8
|
+
import math
|
|
9
|
+
import os
|
|
10
|
+
import signal
|
|
11
|
+
from collections.abc import Mapping
|
|
12
|
+
from pathlib import Path
|
|
13
|
+
from typing import Any
|
|
14
|
+
|
|
15
|
+
from copilot.tools import Tool, ToolInvocation, ToolResult
|
|
16
|
+
|
|
17
|
+
from pytest_copilot.copilot._command_job import CommandJob
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
class CommandCancelledError(asyncio.CancelledError):
|
|
21
|
+
"""Cancellation after cleanup; ``result`` retains partial command output."""
|
|
22
|
+
|
|
23
|
+
def __init__(self, result: ToolResult) -> None:
|
|
24
|
+
super().__init__("Direct command cancelled")
|
|
25
|
+
self.result = result
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
def _result(
|
|
29
|
+
stdout: bytearray,
|
|
30
|
+
stderr: bytearray,
|
|
31
|
+
exit_code: int | None,
|
|
32
|
+
error: str | None = None,
|
|
33
|
+
error_kind: str | None = None,
|
|
34
|
+
) -> ToolResult:
|
|
35
|
+
payload: dict[str, str | int | None] = {
|
|
36
|
+
"exit_code": exit_code,
|
|
37
|
+
"error": error,
|
|
38
|
+
"error_kind": error_kind,
|
|
39
|
+
}
|
|
40
|
+
for label, output in (("stdout", stdout), ("stderr", stderr)):
|
|
41
|
+
try:
|
|
42
|
+
payload[label] = output.decode("utf-8")
|
|
43
|
+
except UnicodeDecodeError:
|
|
44
|
+
payload[label] = output.decode("utf-8", errors="replace")
|
|
45
|
+
payload[f"{label}_base64"] = base64.b64encode(output).decode("ascii")
|
|
46
|
+
return ToolResult(
|
|
47
|
+
text_result_for_llm=json.dumps(payload, ensure_ascii=False),
|
|
48
|
+
result_type="failure" if error is not None else "success",
|
|
49
|
+
error=error,
|
|
50
|
+
)
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
def _text(value: object, label: str, *, empty: bool = True) -> str:
|
|
54
|
+
if not isinstance(value, str) or "\0" in value or (not empty and not value.strip()):
|
|
55
|
+
raise ValueError(f"{label} must be a {'nonempty ' if not empty else ''}string without NUL")
|
|
56
|
+
return value
|
|
57
|
+
|
|
58
|
+
|
|
59
|
+
def create_command_tool(
|
|
60
|
+
executable: str,
|
|
61
|
+
*,
|
|
62
|
+
name: str = "run_command",
|
|
63
|
+
cwd: str | Path | None = None,
|
|
64
|
+
env: Mapping[str, str] | None = None,
|
|
65
|
+
timeout_s: float = 30.0,
|
|
66
|
+
cleanup_timeout_s: float = 5.0,
|
|
67
|
+
terminate_descendants: bool = True,
|
|
68
|
+
) -> Tool:
|
|
69
|
+
"""Create an SDK tool for a caller-selected executable and model-supplied arguments.
|
|
70
|
+
|
|
71
|
+
``cwd`` defaults to the caller's directory at construction. ``env`` overrides
|
|
72
|
+
the environment inherited at invocation; overrides are copied at construction.
|
|
73
|
+
``terminate_descendants=False`` stops only the command process, leaving
|
|
74
|
+
caller-owned services for consumer cleanup. These settings are not model input.
|
|
75
|
+
Input is ``{"args": ["--help"], "stdin": "optional UTF-8 text"}``.
|
|
76
|
+
No argument is parsed as shell syntax. Use a real executable, not a batch
|
|
77
|
+
file or a shell launcher. Caller-owned adapters must enforce application
|
|
78
|
+
permissions; selecting an executable is not a security sandbox.
|
|
79
|
+
|
|
80
|
+
Output is JSON with stdout, stderr, exit_code, error and error_kind. A stream
|
|
81
|
+
with invalid UTF-8 also has a base64 field preserving its exact bytes.
|
|
82
|
+
Nonzero exit, invalid input, startup failure,
|
|
83
|
+
timeout and cleanup failure are tool failures. Cancellation propagates as
|
|
84
|
+
``CommandCancelledError`` with partial output in ``result``.
|
|
85
|
+
"""
|
|
86
|
+
executable = _text(executable, "executable", empty=False)
|
|
87
|
+
name = _text(name, "name", empty=False)
|
|
88
|
+
if type(terminate_descendants) is not bool:
|
|
89
|
+
raise ValueError("terminate_descendants must be a boolean")
|
|
90
|
+
if Path(executable).suffix.lower() in {".bat", ".cmd"}:
|
|
91
|
+
raise ValueError("Select a native executable, not a .bat or .cmd shell script")
|
|
92
|
+
for label, value in (("timeout_s", timeout_s), ("cleanup_timeout_s", cleanup_timeout_s)):
|
|
93
|
+
if isinstance(value, bool) or not isinstance(value, (float, int)):
|
|
94
|
+
raise ValueError(f"{label} must be a finite positive number")
|
|
95
|
+
if not math.isfinite(value) or value <= 0:
|
|
96
|
+
raise ValueError(f"{label} must be a finite positive number")
|
|
97
|
+
directory = str(Path(cwd).absolute() if cwd is not None else Path.cwd())
|
|
98
|
+
_text(directory, "cwd", empty=False)
|
|
99
|
+
overrides: dict[str, str] = {}
|
|
100
|
+
for key, value in (env or {}).items():
|
|
101
|
+
_text(key, "environment key", empty=False)
|
|
102
|
+
if "=" in key:
|
|
103
|
+
raise ValueError("Environment keys must not contain '='")
|
|
104
|
+
overrides[key.upper() if os.name == "nt" else key] = _text(value, "environment value")
|
|
105
|
+
|
|
106
|
+
async def invoke(invocation: ToolInvocation) -> ToolResult:
|
|
107
|
+
stdout, stderr = bytearray(), bytearray()
|
|
108
|
+
try:
|
|
109
|
+
data: Any = invocation.arguments
|
|
110
|
+
if not isinstance(data, dict) or set(data) - {"args", "stdin"}:
|
|
111
|
+
raise ValueError("Input must be an object with args and optional stdin")
|
|
112
|
+
args = data.get("args")
|
|
113
|
+
if not isinstance(args, list):
|
|
114
|
+
raise ValueError("args must be an array of strings")
|
|
115
|
+
arguments = [_text(arg, "argument") for arg in args]
|
|
116
|
+
stdin = None
|
|
117
|
+
if "stdin" in data:
|
|
118
|
+
if not isinstance(data["stdin"], str):
|
|
119
|
+
raise ValueError("stdin must be a string")
|
|
120
|
+
stdin = data["stdin"].encode("utf-8")
|
|
121
|
+
except ValueError as exc:
|
|
122
|
+
return _result(stdout, stderr, None, str(exc), "invalid_input")
|
|
123
|
+
|
|
124
|
+
process: asyncio.subprocess.Process | None = None
|
|
125
|
+
job: CommandJob | None = None
|
|
126
|
+
job_assigned = False
|
|
127
|
+
tasks: list[asyncio.Task[None]] = []
|
|
128
|
+
|
|
129
|
+
async def read(stream: asyncio.StreamReader, output: bytearray) -> None:
|
|
130
|
+
while chunk := await stream.read(65536):
|
|
131
|
+
output.extend(chunk)
|
|
132
|
+
|
|
133
|
+
async def run() -> None:
|
|
134
|
+
nonlocal process, job, job_assigned
|
|
135
|
+
if os.name == "nt" and terminate_descendants:
|
|
136
|
+
job = CommandJob()
|
|
137
|
+
process = await asyncio.create_subprocess_exec(
|
|
138
|
+
executable,
|
|
139
|
+
*arguments,
|
|
140
|
+
cwd=directory,
|
|
141
|
+
env={**os.environ, **overrides},
|
|
142
|
+
stdin=asyncio.subprocess.PIPE if stdin is not None else asyncio.subprocess.DEVNULL,
|
|
143
|
+
stdout=asyncio.subprocess.PIPE,
|
|
144
|
+
stderr=asyncio.subprocess.PIPE,
|
|
145
|
+
start_new_session=os.name != "nt" and terminate_descendants,
|
|
146
|
+
)
|
|
147
|
+
assert process.stdout is not None and process.stderr is not None
|
|
148
|
+
tasks.extend(
|
|
149
|
+
[
|
|
150
|
+
asyncio.create_task(read(process.stdout, stdout)),
|
|
151
|
+
asyncio.create_task(read(process.stderr, stderr)),
|
|
152
|
+
]
|
|
153
|
+
)
|
|
154
|
+
if job is not None:
|
|
155
|
+
job.assign(process.pid)
|
|
156
|
+
job_assigned = True
|
|
157
|
+
if process.stdin is not None:
|
|
158
|
+
try:
|
|
159
|
+
process.stdin.write(stdin or b"")
|
|
160
|
+
await process.stdin.drain()
|
|
161
|
+
except (BrokenPipeError, ConnectionResetError):
|
|
162
|
+
pass # The exit status and stderr describe early command termination.
|
|
163
|
+
finally:
|
|
164
|
+
process.stdin.close()
|
|
165
|
+
await process.wait()
|
|
166
|
+
await asyncio.gather(*tasks)
|
|
167
|
+
|
|
168
|
+
async def cleanup() -> str | None:
|
|
169
|
+
if process is not None:
|
|
170
|
+
try:
|
|
171
|
+
if job is not None:
|
|
172
|
+
job.close()
|
|
173
|
+
if not job_assigned and process.returncode is None:
|
|
174
|
+
process.kill()
|
|
175
|
+
elif os.name != "nt" and terminate_descendants:
|
|
176
|
+
os.killpg(process.pid, signal.SIGKILL)
|
|
177
|
+
elif process.returncode is None:
|
|
178
|
+
process.kill()
|
|
179
|
+
except ProcessLookupError:
|
|
180
|
+
pass
|
|
181
|
+
except OSError as exc:
|
|
182
|
+
return f"Command cleanup failed: {exc}"
|
|
183
|
+
elif job is not None:
|
|
184
|
+
job.close()
|
|
185
|
+
return None
|
|
186
|
+
|
|
187
|
+
execution = asyncio.create_task(run())
|
|
188
|
+
error: str | None = None
|
|
189
|
+
error_kind: str | None = None
|
|
190
|
+
cancelled = False
|
|
191
|
+
try:
|
|
192
|
+
done, _ = await asyncio.wait({execution}, timeout=timeout_s)
|
|
193
|
+
if not done:
|
|
194
|
+
error, error_kind = f"Command timed out after {timeout_s:g}s", "timeout"
|
|
195
|
+
else:
|
|
196
|
+
await execution
|
|
197
|
+
except asyncio.CancelledError:
|
|
198
|
+
cancelled = True
|
|
199
|
+
error, error_kind = "Command cancelled", "cancelled"
|
|
200
|
+
except (OSError, ValueError) as exc:
|
|
201
|
+
error, error_kind = f"Command could not start or complete: {exc}", "start"
|
|
202
|
+
finally:
|
|
203
|
+
# Keep ownership even when the caller cancels again during shutdown.
|
|
204
|
+
async def finish() -> str | None:
|
|
205
|
+
# asyncio owns and reaps a subprocess if creation itself is cancelled.
|
|
206
|
+
if process is None and not execution.done():
|
|
207
|
+
execution.cancel()
|
|
208
|
+
await asyncio.gather(execution, return_exceptions=True)
|
|
209
|
+
cleanup_error = await cleanup()
|
|
210
|
+
pending = {execution, *tasks}
|
|
211
|
+
if process is not None:
|
|
212
|
+
|
|
213
|
+
async def reap() -> None:
|
|
214
|
+
assert process is not None
|
|
215
|
+
await process.wait()
|
|
216
|
+
|
|
217
|
+
pending.add(asyncio.create_task(reap()))
|
|
218
|
+
_, remaining = await asyncio.wait(pending, timeout=cleanup_timeout_s)
|
|
219
|
+
if remaining:
|
|
220
|
+
for task in remaining:
|
|
221
|
+
task.cancel()
|
|
222
|
+
await asyncio.gather(*remaining, return_exceptions=True)
|
|
223
|
+
cleanup_error = cleanup_error or "Command cleanup exceeded its time limit"
|
|
224
|
+
await asyncio.gather(*pending, return_exceptions=True)
|
|
225
|
+
return cleanup_error
|
|
226
|
+
|
|
227
|
+
finishing = asyncio.create_task(finish())
|
|
228
|
+
while True:
|
|
229
|
+
try:
|
|
230
|
+
cleanup_error = await asyncio.shield(finishing)
|
|
231
|
+
break
|
|
232
|
+
except asyncio.CancelledError:
|
|
233
|
+
cancelled = True
|
|
234
|
+
error, error_kind = "Command cancelled", "cancelled"
|
|
235
|
+
if cleanup_error:
|
|
236
|
+
error = f"{error}; {cleanup_error}" if error else cleanup_error
|
|
237
|
+
error_kind = "cleanup"
|
|
238
|
+
|
|
239
|
+
exit_code = process.returncode if process is not None else None
|
|
240
|
+
if error is None and exit_code != 0:
|
|
241
|
+
error, error_kind = f"Command exited with status {exit_code}", "exit"
|
|
242
|
+
result = _result(stdout, stderr, exit_code, error, error_kind)
|
|
243
|
+
if cancelled:
|
|
244
|
+
raise CommandCancelledError(result)
|
|
245
|
+
return result
|
|
246
|
+
|
|
247
|
+
return Tool(
|
|
248
|
+
name=name,
|
|
249
|
+
description=(
|
|
250
|
+
f"Run {Path(executable).name} directly with an argument array, without a shell. "
|
|
251
|
+
"Use its native --help to discover commands. Use this tool for setup and "
|
|
252
|
+
"supporting commands too. Shell operators and quoting are literal arguments. "
|
|
253
|
+
"Returns stdout, stderr, exit_code and explicit errors."
|
|
254
|
+
),
|
|
255
|
+
parameters={
|
|
256
|
+
"type": "object",
|
|
257
|
+
"properties": {
|
|
258
|
+
"args": {"type": "array", "items": {"type": "string"}},
|
|
259
|
+
"stdin": {"type": "string", "description": "Optional UTF-8 standard input"},
|
|
260
|
+
},
|
|
261
|
+
"required": ["args"],
|
|
262
|
+
"additionalProperties": False,
|
|
263
|
+
},
|
|
264
|
+
handler=invoke,
|
|
265
|
+
)
|