zettcode-engine 0.3.4__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- zettcode/_compat.py +35 -0
- zettcode/budget.py +24 -0
- zettcode/config.py +1088 -0
- zettcode/config_edit.py +457 -0
- zettcode/engine/__init__.py +39 -0
- zettcode/engine/agent/__init__.py +81 -0
- zettcode/engine/agent/base.py +60 -0
- zettcode/engine/agent/context.py +166 -0
- zettcode/engine/agent/exports.py +73 -0
- zettcode/engine/agent/interaction.py +54 -0
- zettcode/engine/agent/sessions.py +218 -0
- zettcode/engine/agent/settings.py +269 -0
- zettcode/engine/agent/turns.py +308 -0
- zettcode/engine/core.py +819 -0
- zettcode/engine/dialogue/__init__.py +6 -0
- zettcode/engine/dialogue/ask.py +113 -0
- zettcode/engine/dialogue/side.py +197 -0
- zettcode/engine/dialogue/title.py +80 -0
- zettcode/engine/export/__init__.py +421 -0
- zettcode/engine/export/trace/script.js +18 -0
- zettcode/engine/export/trace/style.css +136 -0
- zettcode/engine/export/trace/template.html +31 -0
- zettcode/engine/extensions/__init__.py +8 -0
- zettcode/engine/extensions/capabilities.py +68 -0
- zettcode/engine/extensions/compaction.py +259 -0
- zettcode/engine/extensions/context.py +521 -0
- zettcode/engine/extensions/files.py +101 -0
- zettcode/engine/extensions/guidance.py +17 -0
- zettcode/engine/extensions/harness.py +208 -0
- zettcode/engine/extensions/harness_switch.py +18 -0
- zettcode/engine/extensions/mcp_notice.py +49 -0
- zettcode/engine/extensions/relay.py +93 -0
- zettcode/engine/extensions/subagents.py +391 -0
- zettcode/engine/extensions/tool_output.py +153 -0
- zettcode/engine/extensions/truncation.py +156 -0
- zettcode/engine/extensions/usage.py +304 -0
- zettcode/engine/harness_paths.py +74 -0
- zettcode/engine/permission/__init__.py +197 -0
- zettcode/engine/permission/approval.py +66 -0
- zettcode/engine/permission/readonly.py +435 -0
- zettcode/engine/permission/sandbox.py +140 -0
- zettcode/engine/pipeline.py +191 -0
- zettcode/engine/plugins/__init__.py +72 -0
- zettcode/engine/plugins/commands.py +152 -0
- zettcode/engine/plugins/context.py +251 -0
- zettcode/engine/plugins/extension.py +183 -0
- zettcode/engine/plugins/groups.py +74 -0
- zettcode/engine/plugins/loader.py +367 -0
- zettcode/engine/plugins/mixins.py +156 -0
- zettcode/engine/plugins/plugin.py +190 -0
- zettcode/engine/plugins/services.py +68 -0
- zettcode/engine/plugins/settings.py +117 -0
- zettcode/engine/plugins/state.py +64 -0
- zettcode/engine/plugins/switches.py +553 -0
- zettcode/engine/py.typed +0 -0
- zettcode/engine/server_tools.py +179 -0
- zettcode/engine/storage/__init__.py +51 -0
- zettcode/engine/storage/layout.py +161 -0
- zettcode/engine/storage/ledger.py +659 -0
- zettcode/engine/storage/locks.py +182 -0
- zettcode/engine/storage/metadata.py +278 -0
- zettcode/engine/storage/records.py +373 -0
- zettcode/engine/storage/store.py +1056 -0
- zettcode/engine/tools/__init__.py +7 -0
- zettcode/engine/tools/boundary.py +56 -0
- zettcode/engine/tools/files.py +262 -0
- zettcode/engine/tools/images.py +32 -0
- zettcode/engine/tools/output.py +81 -0
- zettcode/engine/tools/shell.py +181 -0
- zettcode/engine/tools/workspace.py +271 -0
- zettcode/error_text.py +217 -0
- zettcode/filelock.py +125 -0
- zettcode/logging_setup.py +196 -0
- zettcode/mention.py +124 -0
- zettcode/modes.py +18 -0
- zettcode/paths.py +113 -0
- zettcode/plugin_tui.py +133 -0
- zettcode/processes.py +126 -0
- zettcode/prompt.py +94 -0
- zettcode/providers.py +215 -0
- zettcode/py.typed +0 -0
- zettcode/registry.py +63 -0
- zettcode/relay.py +267 -0
- zettcode/statusline_parts.py +84 -0
- zettcode/version.py +11 -0
- zettcode/wire.py +30 -0
- zettcode/wording.py +11 -0
- zettcode_engine-0.3.4.dist-info/METADATA +23 -0
- zettcode_engine-0.3.4.dist-info/RECORD +90 -0
- zettcode_engine-0.3.4.dist-info/WHEEL +4 -0
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
"""The coding tools the agent offers, written here: read, search, write, edit and the shell.
|
|
2
|
+
|
|
3
|
+
``files`` holds the six file tools and ``shell`` the seventh; ``workspace``, ``output`` and ``images`` are what they are
|
|
4
|
+
built from; ``boundary`` is the contract the shell tool has with the permission level. Nothing here is re-exported,
|
|
5
|
+
so importing the package costs nothing: ``boundary`` is imported by the permission package at start-up and has to stay
|
|
6
|
+
free of the schema library the tools themselves need.
|
|
7
|
+
"""
|
|
@@ -0,0 +1,56 @@
|
|
|
1
|
+
"""What the shell tool asks of the permission level, and what it asks of a sandbox.
|
|
2
|
+
|
|
3
|
+
Both are abstract base classes, and the permission package subclasses them: the tools sit below it in the import
|
|
4
|
+
order, so the contract has to live down here, and a subclass that forgets a method fails when it is built instead of
|
|
5
|
+
when a command first runs. Nothing in this module imports the framework or a model library, so the permission
|
|
6
|
+
package can import it at start-up for free.
|
|
7
|
+
"""
|
|
8
|
+
|
|
9
|
+
from __future__ import annotations
|
|
10
|
+
|
|
11
|
+
from abc import ABC, abstractmethod
|
|
12
|
+
from pathlib import Path
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
class Sandboxer(ABC):
|
|
16
|
+
"""A sandbox that rewrites a shell command so it runs inside it."""
|
|
17
|
+
|
|
18
|
+
@property
|
|
19
|
+
@abstractmethod
|
|
20
|
+
def available(self) -> bool:
|
|
21
|
+
"""Return whether this machine can sandbox a command at all."""
|
|
22
|
+
|
|
23
|
+
@abstractmethod
|
|
24
|
+
def wrap(self, command: str, cwd: Path | None = None) -> str:
|
|
25
|
+
"""Return ``command`` as a shell line that runs sandboxed, in ``cwd`` or in the working directory.
|
|
26
|
+
|
|
27
|
+
The directory is not a detail: the policy a sandbox writes allows one directory, and a Linux sandbox
|
|
28
|
+
moves the process there itself, so a wrapper that is given the working directory runs the command where
|
|
29
|
+
it is not meant to run.
|
|
30
|
+
|
|
31
|
+
Raises:
|
|
32
|
+
RuntimeError: The machine has no sandbox; check :attr:`available` first.
|
|
33
|
+
"""
|
|
34
|
+
|
|
35
|
+
@abstractmethod
|
|
36
|
+
def hint(self) -> str:
|
|
37
|
+
"""Return the note that goes after a failed sandboxed command, naming the boundary it hit."""
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
class Boundary(ABC):
|
|
41
|
+
"""The decision the shell tool defers to at every command: run it sandboxed, or as it was given.
|
|
42
|
+
|
|
43
|
+
``ShellPolicy`` is the implementation. The decision is read at each call and not when the tool is built, so
|
|
44
|
+
``/permission`` takes effect on the next command of a request that is already running.
|
|
45
|
+
"""
|
|
46
|
+
|
|
47
|
+
@abstractmethod
|
|
48
|
+
def sandbox_now(self) -> Sandboxer | None:
|
|
49
|
+
"""Return the sandbox the next command has to run in, or ``None`` to run it as given.
|
|
50
|
+
|
|
51
|
+
One answer, not a flag and a sandbox read separately: a level changed between the two reads could otherwise
|
|
52
|
+
hand back a flag for one and a sandbox for the other.
|
|
53
|
+
"""
|
|
54
|
+
|
|
55
|
+
|
|
56
|
+
__all__ = ["Boundary", "Sandboxer"]
|
|
@@ -0,0 +1,262 @@
|
|
|
1
|
+
"""The file tools: read, search, write and edit.
|
|
2
|
+
|
|
3
|
+
Each tool is a typed function whose signature is its input schema and whose docstring is what the model is told:
|
|
4
|
+
the first paragraph is the description, ``Args`` the parameters, ``Snippet`` the call shapes, ``Guidelines`` the rules
|
|
5
|
+
that go in the prompt. ``zett_agent.tools.base.tool`` turns one into an :class:`~zett_agent.tools.base.AgentTool`;
|
|
6
|
+
the work is done here, in this repository, and by nothing else.
|
|
7
|
+
|
|
8
|
+
There are two readers and no more. ``read`` returns UTF-8 text as a line range and a PNG, JPEG, GIF or WebP file as an
|
|
9
|
+
image, so one tool covers both. ``read_file`` is the same reader without the pictures, for a model that cannot take
|
|
10
|
+
an image (see :class:`~zettcode.engine.extensions.capabilities.ModelCapabilities`, which swaps one for the other).
|
|
11
|
+
There is no ``view_image``, because ``read`` already shows a picture, and no ``delete_file``: removing a file is a
|
|
12
|
+
shell command and goes through the shell's permission like any other, with no second path around it.
|
|
13
|
+
"""
|
|
14
|
+
|
|
15
|
+
from __future__ import annotations
|
|
16
|
+
|
|
17
|
+
import re
|
|
18
|
+
from pathlib import Path
|
|
19
|
+
from typing import Annotated
|
|
20
|
+
|
|
21
|
+
from pydantic import Field
|
|
22
|
+
from zett_agent.messages import ImageBytesSource, ImageContent, TextContent
|
|
23
|
+
from zett_agent.tools.base import tool
|
|
24
|
+
|
|
25
|
+
from .images import MAX_IMAGE_BYTES, SIGNATURE_BYTES, image_media_type
|
|
26
|
+
from .output import MAX_MATCH_BYTES, MAX_OUTPUT_BYTES, utf8_prefix
|
|
27
|
+
from .workspace import (
|
|
28
|
+
MAX_FILE_BYTES,
|
|
29
|
+
Located,
|
|
30
|
+
Pattern,
|
|
31
|
+
apply_edit,
|
|
32
|
+
read_lines,
|
|
33
|
+
read_text,
|
|
34
|
+
write_text,
|
|
35
|
+
)
|
|
36
|
+
|
|
37
|
+
# The parameter types, named once so that the tools that share one agree on its limits and its schema.
|
|
38
|
+
FilePath = Annotated[str, Field(min_length=1)]
|
|
39
|
+
Expression = Annotated[str, Field(min_length=1)]
|
|
40
|
+
Offset = Annotated[int, Field(ge=1)]
|
|
41
|
+
Limit = Annotated[int, Field(ge=1, le=2_000)]
|
|
42
|
+
MaxResults = Annotated[int, Field(ge=1, le=5_000)]
|
|
43
|
+
OldString = Annotated[str, Field(min_length=1)]
|
|
44
|
+
|
|
45
|
+
#: What a reader hands back: text, or a text line and the picture it describes.
|
|
46
|
+
ReadResult = str | list[TextContent | ImageContent]
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
def _image_type_of(path: Path) -> str | None:
|
|
50
|
+
"""Return the media type of a local image file, or ``None`` when it is not a PNG, JPEG, GIF or WebP."""
|
|
51
|
+
with path.open("rb") as source:
|
|
52
|
+
return image_media_type(source.read(SIGNATURE_BYTES))
|
|
53
|
+
|
|
54
|
+
|
|
55
|
+
@tool
|
|
56
|
+
def read(file_path: FilePath, offset: Offset = 1, limit: Limit = 200) -> ReadResult:
|
|
57
|
+
"""Read a local file: UTF-8 text as a line range, or a PNG, JPEG, GIF, or WebP image as an image.
|
|
58
|
+
|
|
59
|
+
Args:
|
|
60
|
+
file_path: Absolute path or a path relative to the current working directory.
|
|
61
|
+
offset: One-based line number to start reading from; ignored for images.
|
|
62
|
+
limit: Maximum number of lines to return; ignored for images.
|
|
63
|
+
|
|
64
|
+
Snippet:
|
|
65
|
+
read(file_path="docs/diagram.png")
|
|
66
|
+
|
|
67
|
+
Guidelines:
|
|
68
|
+
- Use for any local file when you do not know whether it is text or an image.
|
|
69
|
+
- A text file comes back as a line range; use line ranges for large files.
|
|
70
|
+
- A trailing hint says how to go on: the next offset when more lines remain, or the shell when one line is
|
|
71
|
+
longer than the output limit.
|
|
72
|
+
- An image comes back as an image with a one-line description; the range arguments do not apply to it.
|
|
73
|
+
- PDF and SVG files are not supported.
|
|
74
|
+
"""
|
|
75
|
+
located = Located.of(file_path)
|
|
76
|
+
target = located.require_file()
|
|
77
|
+
media_type = _image_type_of(target)
|
|
78
|
+
if media_type is None:
|
|
79
|
+
return read_lines(located, offset, limit)
|
|
80
|
+
size = target.stat().st_size
|
|
81
|
+
if size > MAX_IMAGE_BYTES:
|
|
82
|
+
raise ValueError(f"Image exceeds the {MAX_IMAGE_BYTES}-byte limit: {file_path}")
|
|
83
|
+
return [
|
|
84
|
+
TextContent(text=f"Image {target.name}: {media_type}, {size} bytes"),
|
|
85
|
+
ImageContent(source=ImageBytesSource(data=target.read_bytes(), media_type=media_type), alt_text=target.name),
|
|
86
|
+
]
|
|
87
|
+
|
|
88
|
+
|
|
89
|
+
@tool
|
|
90
|
+
def read_file(file_path: FilePath, offset: Offset = 1, limit: Limit = 200) -> str:
|
|
91
|
+
"""Read a line range from a UTF-8 text file.
|
|
92
|
+
|
|
93
|
+
Args:
|
|
94
|
+
file_path: Absolute path or a path relative to the current working directory.
|
|
95
|
+
offset: One-based line number to start reading from.
|
|
96
|
+
limit: Maximum number of lines to return.
|
|
97
|
+
|
|
98
|
+
Snippet:
|
|
99
|
+
read_file(file_path="src/app.py", offset=1, limit=200)
|
|
100
|
+
|
|
101
|
+
Guidelines:
|
|
102
|
+
- Use line ranges for large files.
|
|
103
|
+
- Inspect the current content before editing a file.
|
|
104
|
+
- A trailing hint says how to go on: the next offset when more lines remain, or the shell when one line is
|
|
105
|
+
longer than the output limit.
|
|
106
|
+
"""
|
|
107
|
+
located = Located.of(file_path)
|
|
108
|
+
located.require_file()
|
|
109
|
+
return read_lines(located, offset, limit)
|
|
110
|
+
|
|
111
|
+
|
|
112
|
+
@tool
|
|
113
|
+
def glob(pattern: Expression, max_results: MaxResults = 200) -> str:
|
|
114
|
+
"""Find paths matching a relative or absolute glob pattern.
|
|
115
|
+
|
|
116
|
+
Args:
|
|
117
|
+
pattern: Glob pattern such as **/*.py, ../tests/*.py, or /work/src/**/*.py.
|
|
118
|
+
max_results: Maximum number of sorted paths to return.
|
|
119
|
+
|
|
120
|
+
Snippet:
|
|
121
|
+
glob(pattern="src/**/*.py", max_results=200)
|
|
122
|
+
|
|
123
|
+
Guidelines:
|
|
124
|
+
- Use glob to discover files before reading or searching them.
|
|
125
|
+
- Narrow the pattern when the result is truncated.
|
|
126
|
+
- The result lists one path per line and is empty when nothing matches.
|
|
127
|
+
"""
|
|
128
|
+
spec = Pattern.parse(pattern)
|
|
129
|
+
paths: list[str] = []
|
|
130
|
+
used = 0
|
|
131
|
+
truncated = False
|
|
132
|
+
for match in spec.matches():
|
|
133
|
+
rendered = spec.render(match)
|
|
134
|
+
size = len(rendered.encode("utf-8"))
|
|
135
|
+
if len(paths) == max_results or used + size > MAX_OUTPUT_BYTES:
|
|
136
|
+
truncated = True
|
|
137
|
+
break
|
|
138
|
+
paths.append(rendered)
|
|
139
|
+
used += size
|
|
140
|
+
result = "\n".join(paths)
|
|
141
|
+
if truncated:
|
|
142
|
+
result += "\n... [truncated; narrow the pattern or increase max_results]"
|
|
143
|
+
return result
|
|
144
|
+
|
|
145
|
+
|
|
146
|
+
@tool
|
|
147
|
+
def grep(
|
|
148
|
+
pattern: Expression,
|
|
149
|
+
file_pattern: Expression = "**/*",
|
|
150
|
+
case_sensitive: bool = True,
|
|
151
|
+
max_results: MaxResults = 200,
|
|
152
|
+
) -> str:
|
|
153
|
+
"""Search UTF-8 text files selected by a relative or absolute glob pattern.
|
|
154
|
+
|
|
155
|
+
Args:
|
|
156
|
+
pattern: Python regular expression to search for on each line.
|
|
157
|
+
file_pattern: Relative or absolute glob selecting files to search.
|
|
158
|
+
case_sensitive: Whether letter case must match exactly.
|
|
159
|
+
max_results: Maximum number of matching lines to return.
|
|
160
|
+
|
|
161
|
+
Snippet:
|
|
162
|
+
grep(pattern="class\\s+Agent", file_pattern="src/**/*.py", max_results=200)
|
|
163
|
+
|
|
164
|
+
Guidelines:
|
|
165
|
+
- Use a narrow file_pattern to avoid scanning unrelated files.
|
|
166
|
+
- Escape regular-expression characters when searching for literal text.
|
|
167
|
+
- Each match is one path:line:text line; use the read tool for surrounding context.
|
|
168
|
+
"""
|
|
169
|
+
spec = Pattern.parse(file_pattern)
|
|
170
|
+
try:
|
|
171
|
+
expression = re.compile(pattern, 0 if case_sensitive else re.IGNORECASE)
|
|
172
|
+
except re.error as error:
|
|
173
|
+
raise ValueError(f"Invalid regular expression: {error}") from error
|
|
174
|
+
lines: list[str] = []
|
|
175
|
+
used = 0
|
|
176
|
+
truncated = False
|
|
177
|
+
for candidate in spec.matches():
|
|
178
|
+
if not candidate.is_file() or candidate.stat().st_size > MAX_FILE_BYTES:
|
|
179
|
+
continue
|
|
180
|
+
try:
|
|
181
|
+
content = candidate.read_text(encoding="utf-8")
|
|
182
|
+
except UnicodeDecodeError:
|
|
183
|
+
continue
|
|
184
|
+
shown = spec.render(candidate)
|
|
185
|
+
for number, line in enumerate(content.splitlines(), start=1):
|
|
186
|
+
found = expression.search(line)
|
|
187
|
+
if found is None:
|
|
188
|
+
continue
|
|
189
|
+
if len(lines) == max_results:
|
|
190
|
+
truncated = True
|
|
191
|
+
break
|
|
192
|
+
first = max(0, found.start() - 100) if len(line.encode("utf-8")) > MAX_MATCH_BYTES else 0
|
|
193
|
+
rendered = f"{shown}:{number}: {utf8_prefix(line[first:], MAX_MATCH_BYTES)}"
|
|
194
|
+
size = len(rendered.encode("utf-8"))
|
|
195
|
+
if used + size > MAX_OUTPUT_BYTES:
|
|
196
|
+
truncated = True
|
|
197
|
+
break
|
|
198
|
+
lines.append(rendered)
|
|
199
|
+
used += size
|
|
200
|
+
if truncated:
|
|
201
|
+
break
|
|
202
|
+
result = "\n".join(lines)
|
|
203
|
+
if truncated:
|
|
204
|
+
result += "\n... [truncated; narrow the search or increase max_results]"
|
|
205
|
+
return result
|
|
206
|
+
|
|
207
|
+
|
|
208
|
+
@tool
|
|
209
|
+
def write_file(file_path: FilePath, content: str, overwrite: bool = True) -> str:
|
|
210
|
+
"""Atomically write a UTF-8 text file.
|
|
211
|
+
|
|
212
|
+
Args:
|
|
213
|
+
file_path: Absolute path or a path relative to the current working directory.
|
|
214
|
+
content: Complete UTF-8 text to write.
|
|
215
|
+
overwrite: Whether an existing file may be replaced.
|
|
216
|
+
|
|
217
|
+
Snippet:
|
|
218
|
+
write_file(file_path="notes/plan.md", content="# Plan\\n")
|
|
219
|
+
|
|
220
|
+
Guidelines:
|
|
221
|
+
- Use for new files or intentional full-file replacement.
|
|
222
|
+
- Prefer edit for a small change to an existing file.
|
|
223
|
+
"""
|
|
224
|
+
located = Located.of(file_path)
|
|
225
|
+
exists = located.path.exists()
|
|
226
|
+
if exists and not located.path.is_file():
|
|
227
|
+
raise ValueError(f"Path is not a file: {file_path}")
|
|
228
|
+
if exists and not overwrite:
|
|
229
|
+
raise ValueError(f"File already exists: {file_path}")
|
|
230
|
+
write_text(located.path, content)
|
|
231
|
+
return f"{'Wrote' if exists else 'Created'} {located.shown}"
|
|
232
|
+
|
|
233
|
+
|
|
234
|
+
@tool
|
|
235
|
+
def edit(file_path: FilePath, old_string: OldString, new_string: str, replace_all: bool = False) -> str:
|
|
236
|
+
"""Edit an existing UTF-8 file by replacing exact text; always use this tool to modify a file.
|
|
237
|
+
|
|
238
|
+
Args:
|
|
239
|
+
file_path: Absolute path or a path relative to the current working directory.
|
|
240
|
+
old_string: The exact text to replace, copied from the file as it is now.
|
|
241
|
+
new_string: The text to put in its place; an empty string deletes the match.
|
|
242
|
+
replace_all: Replace every occurrence of old_string instead of insisting it appears once.
|
|
243
|
+
|
|
244
|
+
Snippet:
|
|
245
|
+
edit(file_path="notes/plan.md", old_string="Draft", new_string="Final")
|
|
246
|
+
edit(file_path="notes/plan.md", old_string="Total", new_string="Sum", replace_all=True)
|
|
247
|
+
|
|
248
|
+
Guidelines:
|
|
249
|
+
- Always use this tool when you need to modify an existing file, instead of sed, awk, perl -i, python -c, tee, or shell redirection. Reserve those for bulk or mechanical rewrites this tool cannot express, such as renaming a symbol across many files.
|
|
250
|
+
- One change a call: call edit again for the next change, and nothing is written when the old text does not match.
|
|
251
|
+
- Avoid sed, cat, and similar shell commands for reading or changing files unless it is necessary; inspect files with the read tool and change them with this tool.
|
|
252
|
+
- Read the target lines first and copy old_string exactly: same indentation, quotes, and line breaks.
|
|
253
|
+
- Make old_string unique by including a neighbouring line or two; set replace_all only when every occurrence must change.
|
|
254
|
+
- When an edit fails, read the file again and retry with corrected old_string rather than switching to a shell command.
|
|
255
|
+
"""
|
|
256
|
+
located = Located.of(file_path)
|
|
257
|
+
updated, replacements = apply_edit(read_text(located.require_file()), old_string, new_string, replace_all)
|
|
258
|
+
write_text(located.path, updated)
|
|
259
|
+
return f"Replaced {replacements} occurrence{'s' if replacements != 1 else ''} in {located.shown}"
|
|
260
|
+
|
|
261
|
+
|
|
262
|
+
__all__ = ["edit", "glob", "grep", "read", "read_file", "write_file"]
|
|
@@ -0,0 +1,32 @@
|
|
|
1
|
+
"""Recognising the image files ``read`` hands to a model: by their first bytes, never by their name."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
#: The largest image file ``read`` returns.
|
|
6
|
+
MAX_IMAGE_BYTES = 10_000_000
|
|
7
|
+
|
|
8
|
+
#: Bytes needed to recognise every signature below.
|
|
9
|
+
SIGNATURE_BYTES = 12
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
def image_media_type(header: bytes) -> str | None:
|
|
13
|
+
"""Return the media type of a PNG, JPEG, GIF or WebP from the first bytes of a file, or ``None`` for anything else.
|
|
14
|
+
|
|
15
|
+
A file's name says what its author meant; its first bytes say what it is, and a model rejects a picture
|
|
16
|
+
announced as the wrong type. A file that is not one of the four is text as far as ``read`` is concerned.
|
|
17
|
+
|
|
18
|
+
Args:
|
|
19
|
+
header: At least :data:`SIGNATURE_BYTES` bytes from the start of the file (fewer for a short file).
|
|
20
|
+
"""
|
|
21
|
+
if header.startswith(b"\x89PNG\r\n\x1a\n"):
|
|
22
|
+
return "image/png"
|
|
23
|
+
if header.startswith(b"\xff\xd8\xff"):
|
|
24
|
+
return "image/jpeg"
|
|
25
|
+
if header.startswith((b"GIF87a", b"GIF89a")):
|
|
26
|
+
return "image/gif"
|
|
27
|
+
if header.startswith(b"RIFF") and header[8:12] == b"WEBP":
|
|
28
|
+
return "image/webp"
|
|
29
|
+
return None
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
__all__ = ["MAX_IMAGE_BYTES", "SIGNATURE_BYTES", "image_media_type"]
|
|
@@ -0,0 +1,81 @@
|
|
|
1
|
+
"""Bounded UTF-8 previews: a long text is cut to a budget, and the full output stays in a file for later reads."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from pathlib import Path
|
|
6
|
+
from typing import NamedTuple
|
|
7
|
+
|
|
8
|
+
#: The most bytes, lines and bytes-per-matching-line any one tool result may carry.
|
|
9
|
+
MAX_OUTPUT_BYTES = 50 * 1024
|
|
10
|
+
MAX_OUTPUT_LINES = 2000
|
|
11
|
+
MAX_MATCH_BYTES = 2000
|
|
12
|
+
|
|
13
|
+
#: What stands where the middle of a long output was left out.
|
|
14
|
+
OMISSION = "\n... [middle omitted; read the output file for full content] ...\n"
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
class Preview(NamedTuple):
|
|
18
|
+
"""A bounded view of one output stream.
|
|
19
|
+
|
|
20
|
+
Attributes:
|
|
21
|
+
text: What fits the budget.
|
|
22
|
+
truncated: Whether the stream was longer than that, so the full text has to be kept.
|
|
23
|
+
"""
|
|
24
|
+
|
|
25
|
+
text: str
|
|
26
|
+
truncated: bool
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
def utf8_prefix(text: str, budget: int) -> str:
|
|
30
|
+
"""Return the longest prefix of ``text`` whose UTF-8 encoding fits in ``budget`` bytes, never half a character."""
|
|
31
|
+
return text.encode("utf-8")[:budget].decode("utf-8", errors="ignore")
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
def stream_preview(path: Path, marker: str = OMISSION) -> Preview:
|
|
35
|
+
"""Return a bounded preview of a stream written to ``path``: its start and its end, with the middle left out.
|
|
36
|
+
|
|
37
|
+
A quarter of the byte budget goes to the start (what a command printed while it started up) and the rest to
|
|
38
|
+
the end (its final results). Only bounded windows are read from disk, so a gigabyte of output costs what fifty
|
|
39
|
+
kilobytes does, and a partial UTF-8 character at either edge of a window is dropped. The line limit applies to
|
|
40
|
+
the text returned, the marker included. The file is never changed, which is what lets the model read what was
|
|
41
|
+
omitted.
|
|
42
|
+
|
|
43
|
+
Args:
|
|
44
|
+
path: The stream that was written.
|
|
45
|
+
marker: What stands between the two windows. A caller that knows where the whole text is kept puts that in
|
|
46
|
+
the marker; it is written once, where the cut is, so a copy of the default marker inside the text cannot
|
|
47
|
+
be mistaken for the cut.
|
|
48
|
+
"""
|
|
49
|
+
with path.open("rb") as source:
|
|
50
|
+
data = source.read(MAX_OUTPUT_BYTES + 1)
|
|
51
|
+
text = data.decode("utf-8", errors="replace")
|
|
52
|
+
if (
|
|
53
|
+
len(data) <= MAX_OUTPUT_BYTES
|
|
54
|
+
and len(text.encode("utf-8")) <= MAX_OUTPUT_BYTES
|
|
55
|
+
and len(text.splitlines()) <= MAX_OUTPUT_LINES
|
|
56
|
+
):
|
|
57
|
+
return Preview(text, False)
|
|
58
|
+
budget = max(2, MAX_OUTPUT_BYTES - len(marker.encode("utf-8")))
|
|
59
|
+
head_budget = budget // 4
|
|
60
|
+
tail_budget = budget - head_budget
|
|
61
|
+
head = data[:head_budget].decode("utf-8", errors="ignore")
|
|
62
|
+
source.seek(max(0, path.stat().st_size - tail_budget))
|
|
63
|
+
tail = source.read(tail_budget).decode("utf-8", errors="ignore")
|
|
64
|
+
# What the marker takes is held back so the line limit covers the text that comes back: a caller's marker is a
|
|
65
|
+
# sentence or two, and the floor keeps a long one from eating the whole preview.
|
|
66
|
+
lines = max(2, MAX_OUTPUT_LINES - max(1, marker.count("\n")) - 1)
|
|
67
|
+
head_lines = max(1, lines // 4)
|
|
68
|
+
head = "".join(head.splitlines(keepends=True)[:head_lines])
|
|
69
|
+
tail = "".join(tail.splitlines(keepends=True)[-(lines - head_lines) :])
|
|
70
|
+
return Preview(head + marker + tail, True)
|
|
71
|
+
|
|
72
|
+
|
|
73
|
+
__all__ = [
|
|
74
|
+
"MAX_MATCH_BYTES",
|
|
75
|
+
"MAX_OUTPUT_BYTES",
|
|
76
|
+
"MAX_OUTPUT_LINES",
|
|
77
|
+
"OMISSION",
|
|
78
|
+
"Preview",
|
|
79
|
+
"stream_preview",
|
|
80
|
+
"utf8_prefix",
|
|
81
|
+
]
|
|
@@ -0,0 +1,181 @@
|
|
|
1
|
+
"""The shell tool: run one command in a directory it names, bounded in time and in what comes back.
|
|
2
|
+
|
|
3
|
+
The command's two streams go to files and not into memory, so the volume of output costs nothing, and what is
|
|
4
|
+
returned is a preview of each (see :func:`~zettcode.engine.tools.output.stream_preview`). When a preview had to cut
|
|
5
|
+
something, the files are kept and the result ends with a note naming them, which the model can read; otherwise they
|
|
6
|
+
are removed. A command that fails or runs out of time is reported as an error that carries its output.
|
|
7
|
+
|
|
8
|
+
At the ``safe`` permission level the command is rewritten to run inside the sandbox (see
|
|
9
|
+
:class:`~zettcode.engine.tools.boundary.Boundary`).
|
|
10
|
+
"""
|
|
11
|
+
|
|
12
|
+
from __future__ import annotations
|
|
13
|
+
|
|
14
|
+
import asyncio
|
|
15
|
+
import os
|
|
16
|
+
import signal
|
|
17
|
+
import tempfile
|
|
18
|
+
from pathlib import Path
|
|
19
|
+
from typing import Annotated
|
|
20
|
+
|
|
21
|
+
from pydantic import Field
|
|
22
|
+
from zett_agent.tools.base import AgentTool, tool
|
|
23
|
+
|
|
24
|
+
from .boundary import Boundary
|
|
25
|
+
from .output import Preview, stream_preview
|
|
26
|
+
from .workspace import Located
|
|
27
|
+
|
|
28
|
+
#: Where a command's kept output goes, inside the working directory. The extension that moves it out of the reader's
|
|
29
|
+
#: project (``ToolOutputExtension``) names the same folder.
|
|
30
|
+
OUTPUT_ROOT = ".zett-tool-output"
|
|
31
|
+
|
|
32
|
+
#: The runtime a command is given unless the model asks for another, and the most it may ask for.
|
|
33
|
+
DEFAULT_TIMEOUT_SECONDS = 60
|
|
34
|
+
MAX_TIMEOUT_SECONDS = 600
|
|
35
|
+
|
|
36
|
+
Command = Annotated[str, Field(min_length=1)]
|
|
37
|
+
WorkDir = Annotated[str, Field(min_length=1)]
|
|
38
|
+
TimeoutSeconds = Annotated[int, Field(ge=1, le=MAX_TIMEOUT_SECONDS)]
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
class CommandFailed(RuntimeError):
|
|
42
|
+
"""A command that exited with a code other than 0, or ran out of time. The message carries what it printed."""
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
def _compose(stdout: Preview, stderr: Preview, kept: tuple[Path, Path] | None, directory: Path) -> str:
|
|
46
|
+
"""Join the two previews into the text a result carries, with the note that names the kept files if there are any."""
|
|
47
|
+
sections: list[str] = []
|
|
48
|
+
if stdout.text:
|
|
49
|
+
sections.append(stdout.text)
|
|
50
|
+
if stderr.text:
|
|
51
|
+
sections.append(f"[stderr]\n{stderr.text}")
|
|
52
|
+
composed = "\n".join(sections)
|
|
53
|
+
if kept is not None:
|
|
54
|
+
first, second = (path.relative_to(directory).as_posix() for path in kept)
|
|
55
|
+
composed += f"\n... [output truncated; full streams kept at {first} and {second}]"
|
|
56
|
+
return composed
|
|
57
|
+
|
|
58
|
+
|
|
59
|
+
async def _stop(process: asyncio.subprocess.Process) -> None:
|
|
60
|
+
"""Kill a command and every process it started, and wait for it to be gone."""
|
|
61
|
+
try:
|
|
62
|
+
if os.name == "posix":
|
|
63
|
+
os.killpg(process.pid, signal.SIGKILL)
|
|
64
|
+
else:
|
|
65
|
+
process.kill()
|
|
66
|
+
except ProcessLookupError:
|
|
67
|
+
pass
|
|
68
|
+
await process.wait()
|
|
69
|
+
|
|
70
|
+
|
|
71
|
+
async def run_command(command: str, timeout_seconds: int, directory: Path | None = None) -> str:
|
|
72
|
+
"""Run ``command`` in ``directory``, or in the working directory, and return what it printed.
|
|
73
|
+
|
|
74
|
+
Raises:
|
|
75
|
+
ValueError: The command is blank.
|
|
76
|
+
CommandFailed: It exited with a code other than 0, or did not finish in ``timeout_seconds``; the message
|
|
77
|
+
carries its output either way.
|
|
78
|
+
"""
|
|
79
|
+
if not command.strip():
|
|
80
|
+
raise ValueError("Shell command cannot be blank")
|
|
81
|
+
root = Located.of(OUTPUT_ROOT)
|
|
82
|
+
# No await between making the root and making the folder in it: the extension that tidies an empty root away
|
|
83
|
+
# runs on this loop, so it can never see the one without the other.
|
|
84
|
+
root.path.mkdir(parents=True, exist_ok=True)
|
|
85
|
+
folder = Path(tempfile.mkdtemp(prefix="shell-", dir=root.path))
|
|
86
|
+
stdout_file, stderr_file = folder / "stdout.txt", folder / "stderr.txt"
|
|
87
|
+
timed_out = False
|
|
88
|
+
# File descriptors rather than pipes keep memory bounded however much is printed. The two streams stay
|
|
89
|
+
# separate: how they interleaved is not kept.
|
|
90
|
+
with stdout_file.open("wb") as stdout_target, stderr_file.open("wb") as stderr_target:
|
|
91
|
+
process = await asyncio.create_subprocess_shell(
|
|
92
|
+
command,
|
|
93
|
+
cwd=directory if directory is not None else root.directory,
|
|
94
|
+
stdin=asyncio.subprocess.DEVNULL,
|
|
95
|
+
stdout=stdout_target,
|
|
96
|
+
stderr=stderr_target,
|
|
97
|
+
start_new_session=os.name == "posix",
|
|
98
|
+
)
|
|
99
|
+
try:
|
|
100
|
+
await asyncio.wait_for(process.wait(), timeout_seconds)
|
|
101
|
+
# ``asyncio.TimeoutError`` is the builtin from 3.11 on and a class of its own before.
|
|
102
|
+
except (TimeoutError, asyncio.TimeoutError):
|
|
103
|
+
await _stop(process)
|
|
104
|
+
timed_out = True
|
|
105
|
+
except asyncio.CancelledError:
|
|
106
|
+
await _stop(process)
|
|
107
|
+
raise
|
|
108
|
+
stdout, stderr = stream_preview(stdout_file), stream_preview(stderr_file)
|
|
109
|
+
truncated = stdout.truncated or stderr.truncated
|
|
110
|
+
composed = _compose(stdout, stderr, (stdout_file, stderr_file) if truncated else None, root.directory)
|
|
111
|
+
if not truncated:
|
|
112
|
+
stdout_file.unlink()
|
|
113
|
+
stderr_file.unlink()
|
|
114
|
+
folder.rmdir()
|
|
115
|
+
if timed_out:
|
|
116
|
+
plural = "" if timeout_seconds == 1 else "s"
|
|
117
|
+
raise CommandFailed(f"Command timed out after {timeout_seconds} second{plural}\n{composed}")
|
|
118
|
+
code = process.returncode if process.returncode is not None else -1
|
|
119
|
+
if code != 0:
|
|
120
|
+
raise CommandFailed(f"Command exited with code {code}\n{composed}")
|
|
121
|
+
return composed
|
|
122
|
+
|
|
123
|
+
|
|
124
|
+
def shell_tool(boundary: Boundary | None = None) -> AgentTool:
|
|
125
|
+
"""Return the ``bash`` tool.
|
|
126
|
+
|
|
127
|
+
Args:
|
|
128
|
+
boundary: What decides, at each command, whether it runs inside a sandbox. ``None`` runs every command
|
|
129
|
+
as it was given. A command that fails inside a sandbox has a note appended that names the boundary,
|
|
130
|
+
because "Operation not permitted" alone sends the model round the same wall again.
|
|
131
|
+
"""
|
|
132
|
+
|
|
133
|
+
@tool
|
|
134
|
+
async def bash(
|
|
135
|
+
command: Command, work_dir: WorkDir | None = None, timeout_seconds: TimeoutSeconds = DEFAULT_TIMEOUT_SECONDS
|
|
136
|
+
) -> str:
|
|
137
|
+
"""Run a shell command with captured output, in the workspace or in ``work_dir``.
|
|
138
|
+
|
|
139
|
+
Args:
|
|
140
|
+
command: Shell command to execute.
|
|
141
|
+
work_dir: Directory to run in: absolute, a `~` path, or relative to the workspace. Omit it to run in the workspace.
|
|
142
|
+
timeout_seconds: Maximum command runtime in seconds; the default is 60.
|
|
143
|
+
|
|
144
|
+
Snippet:
|
|
145
|
+
bash(command="git status --short", timeout_seconds=60)
|
|
146
|
+
bash(command="npm test", work_dir="clients/vscode")
|
|
147
|
+
|
|
148
|
+
Guidelines:
|
|
149
|
+
- Use for builds, tests, git, and searching the code: rg or grep for text, find or ls for paths. Prefer the read, write_file and edit tools for reading and changing a file.
|
|
150
|
+
- Set work_dir instead of cd: every call starts a new shell, so a cd lasts only that one command.
|
|
151
|
+
- Search with a narrow pattern and a path, and pipe through head when the result may be long.
|
|
152
|
+
- Use only for bounded commands in a trusted working directory.
|
|
153
|
+
- A non-zero exit code or a timeout is reported as an error carrying the output.
|
|
154
|
+
- Output previews keep startup and final results within byte and line limits.
|
|
155
|
+
- When output is truncated, read the retained files named in the hint.
|
|
156
|
+
"""
|
|
157
|
+
if not command.strip():
|
|
158
|
+
raise ValueError("Shell command cannot be blank")
|
|
159
|
+
located = None if work_dir is None else Located.of(work_dir)
|
|
160
|
+
if located is not None and not located.path.is_dir():
|
|
161
|
+
raise ValueError(f"Not a directory: {work_dir}")
|
|
162
|
+
directory = located.path if located is not None else None
|
|
163
|
+
sandbox = boundary.sandbox_now() if boundary is not None else None
|
|
164
|
+
if sandbox is None:
|
|
165
|
+
return await run_command(command, timeout_seconds, directory)
|
|
166
|
+
try:
|
|
167
|
+
return await run_command(sandbox.wrap(command, directory), timeout_seconds, directory)
|
|
168
|
+
except CommandFailed as error:
|
|
169
|
+
raise CommandFailed(f"{error}\n{sandbox.hint()}") from error
|
|
170
|
+
|
|
171
|
+
return bash
|
|
172
|
+
|
|
173
|
+
|
|
174
|
+
__all__ = [
|
|
175
|
+
"DEFAULT_TIMEOUT_SECONDS",
|
|
176
|
+
"MAX_TIMEOUT_SECONDS",
|
|
177
|
+
"OUTPUT_ROOT",
|
|
178
|
+
"CommandFailed",
|
|
179
|
+
"run_command",
|
|
180
|
+
"shell_tool",
|
|
181
|
+
]
|