vision-agent 0.2.117__py3-none-any.whl → 0.2.119__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- vision_agent/agent/agent.py +1 -1
- vision_agent/agent/vision_agent.py +107 -49
- vision_agent/agent/vision_agent_coder.py +46 -23
- vision_agent/agent/vision_agent_prompts.py +43 -22
- vision_agent/clients/landing_public_api.py +2 -2
- vision_agent/lmm/lmm.py +15 -6
- vision_agent/lmm/types.py +3 -1
- vision_agent/tools/__init__.py +2 -2
- vision_agent/tools/meta_tools.py +281 -273
- vision_agent/tools/tools.py +36 -14
- vision_agent/tools/tools_types.py +3 -3
- vision_agent/utils/execute.py +69 -22
- vision_agent/utils/image_utils.py +2 -2
- {vision_agent-0.2.117.dist-info → vision_agent-0.2.119.dist-info}/METADATA +12 -8
- {vision_agent-0.2.117.dist-info → vision_agent-0.2.119.dist-info}/RECORD +17 -17
- {vision_agent-0.2.117.dist-info → vision_agent-0.2.119.dist-info}/LICENSE +0 -0
- {vision_agent-0.2.117.dist-info → vision_agent-0.2.119.dist-info}/WHEEL +0 -0
vision_agent/tools/meta_tools.py
CHANGED
@@ -1,12 +1,17 @@
|
|
1
1
|
import os
|
2
|
+
import pickle as pkl
|
2
3
|
import subprocess
|
4
|
+
import tempfile
|
3
5
|
from pathlib import Path
|
4
|
-
from typing import Any, Dict, List, Union
|
6
|
+
from typing import Any, Dict, List, Optional, Union
|
7
|
+
|
8
|
+
from IPython.display import display
|
5
9
|
|
6
10
|
import vision_agent as va
|
7
11
|
from vision_agent.lmm.types import Message
|
8
12
|
from vision_agent.tools.tool_utils import get_tool_documentation
|
9
13
|
from vision_agent.tools.tools import TOOL_DESCRIPTIONS
|
14
|
+
from vision_agent.utils.execute import Execution, MimeType
|
10
15
|
|
11
16
|
# These tools are adapted from SWE-Agent https://github.com/princeton-nlp/SWE-agent
|
12
17
|
|
@@ -35,97 +40,91 @@ def filter_file(file_name: Union[str, Path]) -> bool:
|
|
35
40
|
)
|
36
41
|
|
37
42
|
|
38
|
-
def
|
39
|
-
"""
|
40
|
-
|
41
|
-
|
42
|
-
|
43
|
-
|
44
|
-
|
45
|
-
|
46
|
-
|
47
|
-
|
48
|
-
|
49
|
-
|
50
|
-
|
51
|
-
|
52
|
-
|
53
|
-
|
54
|
-
|
55
|
-
|
56
|
-
|
43
|
+
def redisplay_results(execution: Execution) -> None:
|
44
|
+
"""This function is used to add previous execution results to the current output.
|
45
|
+
This is handy if you are inside a notebook environment, call it notebook1, and you
|
46
|
+
have a nested notebook environment, call it notebook2, and you want the execution
|
47
|
+
results from notebook2 to be included in the execution results for notebook1.
|
48
|
+
"""
|
49
|
+
for result in execution.results:
|
50
|
+
if result.text is not None:
|
51
|
+
display({MimeType.TEXT_PLAIN: result.text})
|
52
|
+
if result.html is not None:
|
53
|
+
display({MimeType.TEXT_HTML: result.html})
|
54
|
+
if result.markdown is not None:
|
55
|
+
display({MimeType.TEXT_MARKDOWN: result.markdown})
|
56
|
+
if result.svg is not None:
|
57
|
+
display({MimeType.IMAGE_SVG: result.svg})
|
58
|
+
if result.png is not None:
|
59
|
+
display({MimeType.IMAGE_PNG: result.png})
|
60
|
+
if result.jpeg is not None:
|
61
|
+
display({MimeType.IMAGE_JPEG: result.jpeg})
|
62
|
+
if result.mp4 is not None:
|
63
|
+
display({MimeType.VIDEO_MP4_B64: result.mp4})
|
64
|
+
if result.latex is not None:
|
65
|
+
display({MimeType.TEXT_LATEX: result.latex})
|
66
|
+
if result.json is not None:
|
67
|
+
display({MimeType.APPLICATION_JSON: result.json})
|
68
|
+
if result.extra is not None:
|
69
|
+
display(result.extra)
|
70
|
+
|
71
|
+
|
72
|
+
class Artifacts:
|
73
|
+
"""Artifacts is a class that allows you to sync files between a local and remote
|
74
|
+
environment. In our case, the remote environment could be where the VisionAgent is
|
75
|
+
executing code and as the user adds new images, files or modifies files, those
|
76
|
+
need to be in sync with the remote environment the VisionAgent is running in.
|
57
77
|
"""
|
58
78
|
|
59
|
-
|
60
|
-
|
61
|
-
|
62
|
-
|
79
|
+
def __init__(self, remote_save_path: Union[str, Path]) -> None:
|
80
|
+
self.remote_save_path = Path(remote_save_path)
|
81
|
+
self.artifacts: Dict[str, Any] = {}
|
82
|
+
|
83
|
+
self.code_sandbox_runtime = None
|
84
|
+
|
85
|
+
def load(self, file_path: Union[str, Path]) -> None:
|
86
|
+
"""Loads are artifacts into the remote environment. If an artifact value is None
|
87
|
+
it will skip loading it.
|
88
|
+
|
89
|
+
Parameters:
|
90
|
+
file_path (Union[str, Path]): The file path to load the artifacts from
|
91
|
+
"""
|
92
|
+
with open(file_path, "rb") as f:
|
93
|
+
self.artifacts = pkl.load(f)
|
94
|
+
for k, v in self.artifacts.items():
|
95
|
+
if v is not None:
|
96
|
+
mode = "w" if isinstance(v, str) else "wb"
|
97
|
+
with open(self.remote_save_path.parent / k, mode) as f:
|
98
|
+
f.write(v)
|
99
|
+
|
100
|
+
def show(self) -> str:
|
101
|
+
"""Shows the artifacts that have been loaded and their remote save paths."""
|
102
|
+
out_str = "[Artifacts loaded]\n"
|
103
|
+
for k in self.artifacts.keys():
|
104
|
+
out_str += (
|
105
|
+
f"Artifact {k} loaded to {str(self.remote_save_path.parent / k)}\n"
|
63
106
|
)
|
107
|
+
out_str += "[End of artifacts]\n"
|
108
|
+
return out_str
|
109
|
+
|
110
|
+
def save(self, local_path: Optional[Union[str, Path]] = None) -> None:
|
111
|
+
save_path = (
|
112
|
+
Path(local_path) if local_path is not None else self.remote_save_path
|
64
113
|
)
|
65
|
-
|
66
|
-
|
67
|
-
try:
|
68
|
-
fixed_chat: List[Message] = [{"role": "user", "content": chat, "media": media}]
|
69
|
-
response = agent.chat_with_workflow(fixed_chat)
|
70
|
-
code = response["code"]
|
71
|
-
with open(save_file, "w") as f:
|
72
|
-
f.write(code)
|
73
|
-
code_lines = code.splitlines(keepends=True)
|
74
|
-
total_lines = len(code_lines)
|
75
|
-
return view_lines(code_lines, 0, total_lines, save_file, total_lines)
|
76
|
-
except Exception as e:
|
77
|
-
return str(e)
|
78
|
-
|
79
|
-
|
80
|
-
def edit_vision_code(code_file: str, chat_history: List[str], media: List[str]) -> str:
|
81
|
-
"""Edits python code to solve a vision based task.
|
114
|
+
with open(save_path, "wb") as f:
|
115
|
+
pkl.dump(self.artifacts, f)
|
82
116
|
|
83
|
-
|
84
|
-
|
85
|
-
chat_history (List[str]): The chat history to used to generate the code.
|
117
|
+
def __iter__(self) -> Any:
|
118
|
+
return iter(self.artifacts)
|
86
119
|
|
87
|
-
|
88
|
-
|
120
|
+
def __getitem__(self, name: str) -> Any:
|
121
|
+
return self.artifacts[name]
|
89
122
|
|
90
|
-
|
91
|
-
|
92
|
-
>>> edit_vision_code(
|
93
|
-
>>> "code.py",
|
94
|
-
>>> ["Can you detect the dogs in this image?", "Can you use a higher threshold?"],
|
95
|
-
>>> ["dog.jpg"],
|
96
|
-
>>> )
|
97
|
-
from vision_agent.tools import load_image, owl_v2
|
98
|
-
def detect_dogs(image_path: str):
|
99
|
-
image = load_image(image_path)
|
100
|
-
dogs = owl_v2("dog", image, threshold=0.8)
|
101
|
-
return dogs
|
102
|
-
"""
|
123
|
+
def __setitem__(self, name: str, value: Any) -> None:
|
124
|
+
self.artifacts[name] = value
|
103
125
|
|
104
|
-
|
105
|
-
|
106
|
-
code = f.read()
|
107
|
-
|
108
|
-
# Append latest code to second to last message from assistant
|
109
|
-
fixed_chat_history: List[Message] = []
|
110
|
-
for i, chat in enumerate(chat_history):
|
111
|
-
if i == 0:
|
112
|
-
fixed_chat_history.append({"role": "user", "content": chat, "media": media})
|
113
|
-
elif i > 0 and i < len(chat_history) - 1:
|
114
|
-
fixed_chat_history.append({"role": "user", "content": chat})
|
115
|
-
elif i == len(chat_history) - 1:
|
116
|
-
fixed_chat_history.append({"role": "assistant", "content": code})
|
117
|
-
fixed_chat_history.append({"role": "user", "content": chat})
|
118
|
-
|
119
|
-
try:
|
120
|
-
response = agent.chat_with_workflow(fixed_chat_history, test_multi_plan=False)
|
121
|
-
code = response["code"]
|
122
|
-
with open(code_file, "w") as f:
|
123
|
-
f.write(code)
|
124
|
-
code_lines = code.splitlines(keepends=True)
|
125
|
-
total_lines = len(code_lines)
|
126
|
-
return view_lines(code_lines, 0, total_lines, code_file, total_lines)
|
127
|
-
except Exception as e:
|
128
|
-
return str(e)
|
126
|
+
def __contains__(self, name: str) -> bool:
|
127
|
+
return name in self.artifacts
|
129
128
|
|
130
129
|
|
131
130
|
def format_lines(lines: List[str], start_idx: int) -> str:
|
@@ -136,34 +135,40 @@ def format_lines(lines: List[str], start_idx: int) -> str:
|
|
136
135
|
|
137
136
|
|
138
137
|
def view_lines(
|
139
|
-
lines: List[str], line_num: int, window_size: int,
|
138
|
+
lines: List[str], line_num: int, window_size: int, name: str, total_lines: int
|
140
139
|
) -> str:
|
141
140
|
start = max(0, line_num - window_size)
|
142
141
|
end = min(len(lines), line_num + window_size)
|
143
|
-
|
144
|
-
f"[
|
142
|
+
return_str = (
|
143
|
+
f"[Artifact: {name} ({total_lines} lines total)]\n"
|
145
144
|
+ format_lines(lines[start:end], start)
|
146
|
-
+ (
|
145
|
+
+ (
|
146
|
+
"[End of artifact]"
|
147
|
+
if end == len(lines)
|
148
|
+
else f"[{len(lines) - end} more lines]"
|
149
|
+
)
|
147
150
|
)
|
151
|
+
print(return_str)
|
152
|
+
return return_str
|
148
153
|
|
149
154
|
|
150
|
-
def
|
151
|
-
|
152
|
-
|
153
|
-
|
154
|
-
|
155
|
+
def open_code_artifact(
|
156
|
+
artifacts: Artifacts, name: str, line_num: int = 0, window_size: int = 100
|
157
|
+
) -> str:
|
158
|
+
"""Opens the provided code artifact. If `line_num` is provided, the window will be
|
159
|
+
moved to include that line. It only shows the first 100 lines by default! Max
|
160
|
+
`window_size` supported is 2000.
|
155
161
|
|
156
162
|
Parameters:
|
157
|
-
|
163
|
+
artifacts (Artifacts): The artifacts object to open the artifact from.
|
164
|
+
name (str): The name of the artifact to open.
|
158
165
|
line_num (int): The line number to move the window to.
|
159
166
|
window_size (int): The number of lines to show above and below the line.
|
160
167
|
"""
|
168
|
+
if name not in artifacts:
|
169
|
+
return f"[Artifact {name} does not exist]"
|
161
170
|
|
162
|
-
|
163
|
-
if not file_path_p.exists():
|
164
|
-
return f"[File {file_path} does not exist]"
|
165
|
-
|
166
|
-
total_lines = sum(1 for _ in open(file_path_p))
|
171
|
+
total_lines = len(artifacts[name].splitlines())
|
167
172
|
window_size = min(window_size, 2000)
|
168
173
|
window_size = window_size // 2
|
169
174
|
if line_num - window_size < 0:
|
@@ -171,211 +176,218 @@ def open_file(file_path: str, line_num: int = 0, window_size: int = 100) -> str:
|
|
171
176
|
elif line_num >= total_lines:
|
172
177
|
line_num = total_lines - 1 - window_size
|
173
178
|
|
174
|
-
|
175
|
-
CURRENT_LINE = line_num
|
176
|
-
CURRENT_FILE = file_path
|
179
|
+
lines = artifacts[name].splitlines(keepends=True)
|
177
180
|
|
178
|
-
|
179
|
-
lines = f.readlines()
|
181
|
+
return view_lines(lines, line_num, window_size, name, total_lines)
|
180
182
|
|
181
|
-
return view_lines(lines, line_num, window_size, file_path, total_lines)
|
182
183
|
|
183
|
-
|
184
|
-
|
185
|
-
"""Creates and opens a new file with the given name.
|
184
|
+
def create_code_artifact(artifacts: Artifacts, name: str) -> str:
|
185
|
+
"""Creates a new code artifiact with the given name.
|
186
186
|
|
187
187
|
Parameters:
|
188
|
-
|
188
|
+
artifacts (Artifacts): The artifacts object to add the new artifact to.
|
189
|
+
name (str): The name of the new artifact.
|
189
190
|
"""
|
191
|
+
if name in artifacts:
|
192
|
+
return_str = f"[Artifact {name} already exists]"
|
193
|
+
else:
|
194
|
+
artifacts[name] = ""
|
195
|
+
return_str = f"[Artifact {name} created]"
|
196
|
+
print(return_str)
|
190
197
|
|
191
|
-
|
192
|
-
|
193
|
-
return f"[File {file_path} already exists]"
|
194
|
-
file_path_p.touch()
|
195
|
-
global CURRENT_FILE
|
196
|
-
CURRENT_FILE = file_path
|
197
|
-
return f"[File created {file_path}]"
|
198
|
-
|
198
|
+
display({MimeType.APPLICATION_JSON: {"last_artifact": name}})
|
199
|
+
return return_str
|
199
200
|
|
200
|
-
def scroll_up() -> str:
|
201
|
-
"""Moves the window up by 100 lines."""
|
202
|
-
if CURRENT_FILE is None:
|
203
|
-
return "[No file is open]"
|
204
201
|
|
205
|
-
|
202
|
+
def edit_code_artifact(
|
203
|
+
artifacts: Artifacts, name: str, start: int, end: int, content: str
|
204
|
+
) -> str:
|
205
|
+
"""Edits the given code artifact with the provided content. The content will be
|
206
|
+
inserted between the `start` and `end` line numbers. If the `start` and `end` are
|
207
|
+
the same, the content will be inserted at the `start` line number. If the `end` is
|
208
|
+
greater than the total number of lines in the file, the content will be inserted at
|
209
|
+
the end of the file. If the `start` or `end` are negative, the function will return
|
210
|
+
an error message.
|
206
211
|
|
212
|
+
Parameters:
|
213
|
+
artifacts (Artifacts): The artifacts object to edit the artifact from.
|
214
|
+
name (str): The name of the artifact to edit.
|
215
|
+
start (int): The line number to start the edit.
|
216
|
+
end (int): The line number to end the edit.
|
217
|
+
content (str): The content to insert.
|
218
|
+
"""
|
219
|
+
# just make the artifact if it doesn't exist instead of forcing agent to call
|
220
|
+
# create_artifact
|
221
|
+
if name not in artifacts:
|
222
|
+
artifacts[name] = ""
|
207
223
|
|
208
|
-
|
209
|
-
|
210
|
-
|
211
|
-
|
224
|
+
total_lines = len(artifacts[name].splitlines())
|
225
|
+
if start < 0 or end < 0 or start > end or end > total_lines:
|
226
|
+
return "[Invalid line range]"
|
227
|
+
if start == end:
|
228
|
+
end += 1
|
212
229
|
|
213
|
-
|
230
|
+
new_content_lines = content.splitlines(keepends=True)
|
231
|
+
new_content_lines = [
|
232
|
+
line if line.endswith("\n") else line + "\n" for line in new_content_lines
|
233
|
+
]
|
234
|
+
lines = artifacts[name].splitlines()
|
235
|
+
edited_lines = lines[:start] + new_content_lines + lines[end:]
|
214
236
|
|
237
|
+
cur_line = start + len(content.split("\n")) // 2
|
238
|
+
with tempfile.NamedTemporaryFile(delete=True) as f:
|
239
|
+
with open(f.name, "w") as f: # type: ignore
|
240
|
+
f.writelines(edited_lines)
|
241
|
+
|
242
|
+
process = subprocess.Popen(
|
243
|
+
[
|
244
|
+
"flake8",
|
245
|
+
"--isolated",
|
246
|
+
"--select=F821,F822,F831,E111,E112,E113,E999,E902",
|
247
|
+
f.name,
|
248
|
+
],
|
249
|
+
stdout=subprocess.PIPE,
|
250
|
+
stderr=subprocess.PIPE,
|
251
|
+
text=True,
|
252
|
+
)
|
253
|
+
stdout, _ = process.communicate()
|
254
|
+
|
255
|
+
if stdout != "":
|
256
|
+
stdout = stdout.replace(f.name, name)
|
257
|
+
error_msg = "[Edit failed with the following status]\n" + stdout
|
258
|
+
original_view = view_lines(
|
259
|
+
lines,
|
260
|
+
start + ((end - start) // 2),
|
261
|
+
DEFAULT_WINDOW_SIZE,
|
262
|
+
name,
|
263
|
+
total_lines,
|
264
|
+
)
|
265
|
+
total_lines_edit = sum(1 for _ in edited_lines)
|
266
|
+
edited_view = view_lines(
|
267
|
+
edited_lines, cur_line, DEFAULT_WINDOW_SIZE, name, total_lines_edit
|
268
|
+
)
|
215
269
|
|
216
|
-
|
217
|
-
|
270
|
+
error_msg += f"\n[This is how your edit would have looked like if applied]\n{edited_view}\n\n[This is the original code before your edit]\n{original_view}"
|
271
|
+
return error_msg
|
218
272
|
|
219
|
-
|
220
|
-
search_term (str): The search term to look for.
|
221
|
-
dir_path (str): The directory path to search in, preferred absolute path.
|
222
|
-
"""
|
273
|
+
artifacts[name] = "".join(edited_lines)
|
223
274
|
|
224
|
-
|
225
|
-
|
226
|
-
return f"[Directory {dir_path} does not exist]"
|
227
|
-
|
228
|
-
matches = []
|
229
|
-
for file in dir_path_p.glob("**/*"):
|
230
|
-
if filter_file(file):
|
231
|
-
with open(file, "r") as f:
|
232
|
-
lines = f.readlines()
|
233
|
-
for i, line in enumerate(lines):
|
234
|
-
if search_term in line:
|
235
|
-
matches.append(f"{file}:{i}|{line.strip()}\n")
|
236
|
-
if not matches:
|
237
|
-
return f"[No matches found for {search_term} in {dir_path}]"
|
238
|
-
if len(matches) > 100:
|
239
|
-
return f"[More than {len(matches)} matches found for {search_term} in {dir_path}. Please narrow your search]"
|
240
|
-
|
241
|
-
return_str = f"[Found {len(matches)} matches for {search_term} in {dir_path}]\n"
|
242
|
-
for match in matches:
|
243
|
-
return_str += match
|
244
|
-
|
245
|
-
return_str += f"[End of matches for {search_term} in {dir_path}]"
|
246
|
-
return return_str
|
275
|
+
display({MimeType.APPLICATION_JSON: {"last_artifact": name}})
|
276
|
+
return open_code_artifact(artifacts, name, cur_line)
|
247
277
|
|
248
278
|
|
249
|
-
def
|
250
|
-
|
279
|
+
def generate_vision_code(
|
280
|
+
artifacts: Artifacts, name: str, chat: str, media: List[str]
|
281
|
+
) -> str:
|
282
|
+
"""Generates python code to solve vision based tasks.
|
251
283
|
|
252
284
|
Parameters:
|
253
|
-
|
254
|
-
|
255
|
-
|
256
|
-
|
257
|
-
file_path_p = Path(file_path)
|
258
|
-
if not file_path_p.exists():
|
259
|
-
return f"[File {file_path} does not exist]"
|
285
|
+
artifacts (Artifacts): The artifacts object to save the code to.
|
286
|
+
name (str): The name of the artifact to save the code to.
|
287
|
+
chat (str): The chat message from the user.
|
288
|
+
media (List[str]): The media files to use.
|
260
289
|
|
261
|
-
|
262
|
-
|
290
|
+
Returns:
|
291
|
+
str: The generated code.
|
263
292
|
|
264
|
-
|
265
|
-
|
266
|
-
|
267
|
-
|
293
|
+
Examples
|
294
|
+
--------
|
295
|
+
>>> generate_vision_code(artifacts, "code.py", "Can you detect the dogs in this image?", ["image.jpg"])
|
296
|
+
from vision_agent.tools import load_image, owl_v2
|
297
|
+
def detect_dogs(image_path: str):
|
298
|
+
image = load_image(image_path)
|
299
|
+
dogs = owl_v2("dog", image)
|
300
|
+
return dogs
|
301
|
+
"""
|
268
302
|
|
269
|
-
if not
|
270
|
-
|
303
|
+
if ZMQ_PORT is not None:
|
304
|
+
agent = va.agent.VisionAgentCoder(
|
305
|
+
report_progress_callback=lambda inp: report_progress_callback(
|
306
|
+
int(ZMQ_PORT), inp
|
307
|
+
)
|
308
|
+
)
|
309
|
+
else:
|
310
|
+
agent = va.agent.VisionAgentCoder()
|
271
311
|
|
272
|
-
|
273
|
-
|
274
|
-
)
|
275
|
-
|
276
|
-
|
312
|
+
fixed_chat: List[Message] = [{"role": "user", "content": chat, "media": media}]
|
313
|
+
response = agent.chat_with_workflow(fixed_chat, test_multi_plan=True)
|
314
|
+
redisplay_results(response["test_result"])
|
315
|
+
code = response["code"]
|
316
|
+
artifacts[name] = code
|
317
|
+
code_lines = code.splitlines(keepends=True)
|
318
|
+
total_lines = len(code_lines)
|
277
319
|
|
278
|
-
|
279
|
-
return
|
320
|
+
display({MimeType.APPLICATION_JSON: {"last_artifact": name}})
|
321
|
+
return view_lines(code_lines, 0, total_lines, name, total_lines)
|
280
322
|
|
281
323
|
|
282
|
-
def
|
283
|
-
|
324
|
+
def edit_vision_code(
|
325
|
+
artifacts: Artifacts, name: str, chat_history: List[str], media: List[str]
|
326
|
+
) -> str:
|
327
|
+
"""Edits python code to solve a vision based task.
|
284
328
|
|
285
329
|
Parameters:
|
286
|
-
|
287
|
-
|
288
|
-
|
289
|
-
|
290
|
-
dir_path_p = Path(dir_path)
|
291
|
-
if not dir_path_p.exists():
|
292
|
-
return f"[Directory {dir_path} does not exist]"
|
293
|
-
|
294
|
-
files = list(dir_path_p.glob(f"**/*{file_name}*"))
|
295
|
-
files = [f for f in files if filter_file(f)]
|
296
|
-
if not files:
|
297
|
-
return f"[No files found in {dir_path} with name {file_name}]"
|
298
|
-
|
299
|
-
return_str = f"[Found {len(files)} matches for {file_name} in {dir_path}]\n"
|
300
|
-
for match in files:
|
301
|
-
return_str += str(match) + "\n"
|
330
|
+
artifacts (Artifacts): The artifacts object to save the code to.
|
331
|
+
name (str): The file path to the code.
|
332
|
+
chat_history (List[str]): The chat history to used to generate the code.
|
302
333
|
|
303
|
-
|
304
|
-
|
334
|
+
Returns:
|
335
|
+
str: The edited code.
|
305
336
|
|
337
|
+
Examples
|
338
|
+
--------
|
339
|
+
>>> edit_vision_code(
|
340
|
+
>>> artifacts,
|
341
|
+
>>> "code.py",
|
342
|
+
>>> ["Can you detect the dogs in this image?", "Can you use a higher threshold?"],
|
343
|
+
>>> ["dog.jpg"],
|
344
|
+
>>> )
|
345
|
+
from vision_agent.tools import load_image, owl_v2
|
346
|
+
def detect_dogs(image_path: str):
|
347
|
+
image = load_image(image_path)
|
348
|
+
dogs = owl_v2("dog", image, threshold=0.8)
|
349
|
+
return dogs
|
350
|
+
"""
|
306
351
|
|
307
|
-
|
308
|
-
|
309
|
-
|
310
|
-
the same, the content will be inserted at the `start` line number. If the `end` is
|
311
|
-
greater than the total number of lines in the file, the content will be inserted at
|
312
|
-
the end of the file. If the `start` or `end` are negative, the function will return
|
313
|
-
an error message.
|
352
|
+
agent = va.agent.VisionAgentCoder()
|
353
|
+
if name not in artifacts:
|
354
|
+
return f"[Artifact {name} does not exist]"
|
314
355
|
|
315
|
-
|
316
|
-
file_path (str): The file path to edit, preferred absolute path.
|
317
|
-
start (int): The line number to start the edit.
|
318
|
-
end (int): The line number to end the edit.
|
319
|
-
content (str): The content to insert.
|
320
|
-
"""
|
321
|
-
file_path_p = Path(file_path)
|
322
|
-
if not file_path_p.exists():
|
323
|
-
return f"[File {file_path} does not exist]"
|
356
|
+
code = artifacts[name]
|
324
357
|
|
325
|
-
|
326
|
-
|
327
|
-
|
328
|
-
|
329
|
-
|
358
|
+
# Append latest code to second to last message from assistant
|
359
|
+
fixed_chat_history: List[Message] = []
|
360
|
+
for i, chat in enumerate(chat_history):
|
361
|
+
if i == 0:
|
362
|
+
fixed_chat_history.append({"role": "user", "content": chat, "media": media})
|
363
|
+
elif i > 0 and i < len(chat_history) - 1:
|
364
|
+
fixed_chat_history.append({"role": "user", "content": chat})
|
365
|
+
elif i == len(chat_history) - 1:
|
366
|
+
fixed_chat_history.append({"role": "assistant", "content": code})
|
367
|
+
fixed_chat_history.append({"role": "user", "content": chat})
|
330
368
|
|
331
|
-
|
332
|
-
|
333
|
-
|
334
|
-
]
|
335
|
-
|
336
|
-
|
337
|
-
edited_lines = lines[:start] + new_content_lines + lines[end:]
|
369
|
+
response = agent.chat_with_workflow(fixed_chat_history, test_multi_plan=False)
|
370
|
+
redisplay_results(response["test_result"])
|
371
|
+
code = response["code"]
|
372
|
+
artifacts[name] = code
|
373
|
+
code_lines = code.splitlines(keepends=True)
|
374
|
+
total_lines = len(code_lines)
|
338
375
|
|
339
|
-
|
340
|
-
|
341
|
-
with open(tmp_file, "w") as f:
|
342
|
-
f.writelines(edited_lines)
|
343
|
-
|
344
|
-
process = subprocess.Popen(
|
345
|
-
[
|
346
|
-
"flake8",
|
347
|
-
"--isolated",
|
348
|
-
"--select=F821,F822,F831,E111,E112,E113,E999,E902",
|
349
|
-
tmp_file,
|
350
|
-
],
|
351
|
-
stdout=subprocess.PIPE,
|
352
|
-
stderr=subprocess.PIPE,
|
353
|
-
text=True,
|
354
|
-
)
|
355
|
-
stdout, _ = process.communicate()
|
356
|
-
tmp_file.unlink()
|
357
|
-
if stdout != "":
|
358
|
-
stdout = stdout.replace(tmp_file.name, file_path)
|
359
|
-
error_msg = "[Edit failed with the following status]\n" + stdout
|
360
|
-
original_view = view_lines(
|
361
|
-
lines,
|
362
|
-
start + ((end - start) // 2),
|
363
|
-
DEFAULT_WINDOW_SIZE,
|
364
|
-
file_path,
|
365
|
-
total_lines,
|
366
|
-
)
|
367
|
-
total_lines_edit = sum(1 for _ in edited_lines)
|
368
|
-
edited_view = view_lines(
|
369
|
-
edited_lines, cur_line, DEFAULT_WINDOW_SIZE, file_path, total_lines_edit
|
370
|
-
)
|
376
|
+
display({MimeType.APPLICATION_JSON: {"last_artifact": name}})
|
377
|
+
return view_lines(code_lines, 0, total_lines, name, total_lines)
|
371
378
|
|
372
|
-
error_msg += f"\n[This is how your edit would have looked like if applied]\n{edited_view}\n\n[This is the original code before your edit]\n{original_view}"
|
373
|
-
return error_msg
|
374
379
|
|
375
|
-
|
376
|
-
|
380
|
+
def write_media_artifact(artifacts: Artifacts, local_path: str) -> str:
|
381
|
+
"""Writes a media file to the artifacts object.
|
377
382
|
|
378
|
-
|
383
|
+
Parameters:
|
384
|
+
artifacts (Artifacts): The artifacts object to save the media to.
|
385
|
+
local_path (str): The local path to the media file.
|
386
|
+
"""
|
387
|
+
with open(local_path, "rb") as f:
|
388
|
+
media = f.read()
|
389
|
+
artifacts[Path(local_path).name] = media
|
390
|
+
return f"[Media {Path(local_path).name} saved]"
|
379
391
|
|
380
392
|
|
381
393
|
def get_tool_descriptions() -> str:
|
@@ -388,15 +400,11 @@ def get_tool_descriptions() -> str:
|
|
388
400
|
META_TOOL_DOCSTRING = get_tool_documentation(
|
389
401
|
[
|
390
402
|
get_tool_descriptions,
|
403
|
+
open_code_artifact,
|
404
|
+
create_code_artifact,
|
405
|
+
edit_code_artifact,
|
391
406
|
generate_vision_code,
|
392
407
|
edit_vision_code,
|
393
|
-
|
394
|
-
create_file,
|
395
|
-
scroll_up,
|
396
|
-
scroll_down,
|
397
|
-
edit_file,
|
398
|
-
search_dir,
|
399
|
-
search_file,
|
400
|
-
find_file,
|
408
|
+
write_media_artifact,
|
401
409
|
]
|
402
410
|
)
|