meshagent-computers 0.0.37__py3-none-any.whl → 0.0.38__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of meshagent-computers might be problematic. Click here for more details.
- meshagent/computers/__init__.py +13 -0
- meshagent/computers/agent.py +98 -91
- meshagent/computers/base_playwright.py +2 -8
- meshagent/computers/browserbase.py +10 -11
- meshagent/computers/computer.py +5 -5
- meshagent/computers/docker.py +9 -9
- meshagent/computers/local_playwright.py +6 -5
- meshagent/computers/operator.py +3 -2
- meshagent/computers/version.py +1 -1
- meshagent_computers-0.0.38.dist-info/METADATA +65 -0
- meshagent_computers-0.0.38.dist-info/RECORD +16 -0
- meshagent_computers-0.0.37.dist-info/METADATA +0 -23
- meshagent_computers-0.0.37.dist-info/RECORD +0 -16
- {meshagent_computers-0.0.37.dist-info → meshagent_computers-0.0.38.dist-info}/WHEEL +0 -0
- {meshagent_computers-0.0.37.dist-info → meshagent_computers-0.0.38.dist-info}/licenses/LICENSE +0 -0
- {meshagent_computers-0.0.37.dist-info → meshagent_computers-0.0.38.dist-info}/top_level.txt +0 -0
meshagent/computers/__init__.py
CHANGED
|
@@ -6,3 +6,16 @@ from .scrapybara import ScrapybaraBrowser, ScrapybaraUbuntu
|
|
|
6
6
|
from .operator import Operator
|
|
7
7
|
from .agent import ComputerAgent
|
|
8
8
|
from .version import __version__
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
__all__ = [
|
|
12
|
+
Computer,
|
|
13
|
+
BrowserbaseBrowser,
|
|
14
|
+
LocalPlaywrightComputer,
|
|
15
|
+
DockerComputer,
|
|
16
|
+
ScrapybaraBrowser,
|
|
17
|
+
ScrapybaraUbuntu,
|
|
18
|
+
Operator,
|
|
19
|
+
ComputerAgent,
|
|
20
|
+
__version__,
|
|
21
|
+
]
|
meshagent/computers/agent.py
CHANGED
|
@@ -3,7 +3,6 @@ from meshagent.tools import Tool, Toolkit, ToolContext
|
|
|
3
3
|
from meshagent.computers import Computer, Operator, BrowserbaseBrowser
|
|
4
4
|
from meshagent.agents.chat import ChatBot, ChatThreadContext
|
|
5
5
|
from meshagent.api import RequiredToolkit, RemoteParticipant
|
|
6
|
-
from meshagent.api.messaging import RawOutputs
|
|
7
6
|
from meshagent.tools.toolkit import register_toolkit_factory
|
|
8
7
|
from meshagent.openai.tools.responses_adapter import OpenAIResponsesTool
|
|
9
8
|
|
|
@@ -14,8 +13,11 @@ import logging
|
|
|
14
13
|
logger = logging.getLogger("computer")
|
|
15
14
|
logger.setLevel(logging.WARN)
|
|
16
15
|
|
|
16
|
+
|
|
17
17
|
class ComputerToolkit(Toolkit):
|
|
18
|
-
def __init__(
|
|
18
|
+
def __init__(
|
|
19
|
+
self, name: str, computer: Computer, operator: Operator, tools: list[Tool]
|
|
20
|
+
):
|
|
19
21
|
super().__init__(name=name, tools=tools)
|
|
20
22
|
self.computer = computer
|
|
21
23
|
self.operator = operator
|
|
@@ -23,74 +25,80 @@ class ComputerToolkit(Toolkit):
|
|
|
23
25
|
self._starting = None
|
|
24
26
|
|
|
25
27
|
async def ensure_started(self):
|
|
26
|
-
|
|
27
28
|
self.started = False
|
|
28
29
|
|
|
29
|
-
if self.started
|
|
30
|
-
|
|
30
|
+
if not self.started:
|
|
31
31
|
self.started = True
|
|
32
32
|
await self.computer.__aenter__()
|
|
33
|
-
|
|
34
33
|
|
|
35
34
|
|
|
36
|
-
def make_computer_toolkit(
|
|
37
|
-
|
|
35
|
+
def make_computer_toolkit(
|
|
36
|
+
*,
|
|
37
|
+
operator_cls: Type[Operator],
|
|
38
|
+
computer_cls: Type[Computer],
|
|
39
|
+
render_screen: Callable[[bytes], None],
|
|
40
|
+
):
|
|
38
41
|
operator = operator_cls()
|
|
39
42
|
computer = computer_cls()
|
|
40
|
-
|
|
41
43
|
|
|
42
44
|
class ComputerTool(OpenAIResponsesTool):
|
|
43
|
-
def __init__(
|
|
45
|
+
def __init__(
|
|
46
|
+
self,
|
|
47
|
+
*,
|
|
48
|
+
operator: Operator,
|
|
49
|
+
computer: Computer,
|
|
50
|
+
title="computer_call",
|
|
51
|
+
description="handle computer calls from computer use preview",
|
|
52
|
+
rules=[],
|
|
53
|
+
thumbnail_url=None,
|
|
54
|
+
):
|
|
44
55
|
super().__init__(
|
|
45
56
|
name="computer_call",
|
|
46
57
|
# TODO: give a correct schema
|
|
47
58
|
title=title,
|
|
48
59
|
description=description,
|
|
49
60
|
rules=rules,
|
|
50
|
-
thumbnail_url=thumbnail_url,
|
|
61
|
+
thumbnail_url=thumbnail_url,
|
|
51
62
|
)
|
|
52
63
|
self.computer = computer
|
|
53
64
|
|
|
54
65
|
def get_open_ai_tool_definitions(self) -> list[dict]:
|
|
55
|
-
return [
|
|
66
|
+
return [
|
|
56
67
|
{
|
|
57
68
|
"type": "computer_use_preview",
|
|
58
69
|
"display_width": self.computer.dimensions[0],
|
|
59
70
|
"display_height": self.computer.dimensions[1],
|
|
60
71
|
"environment": self.computer.environment,
|
|
61
|
-
}
|
|
62
|
-
|
|
63
|
-
|
|
72
|
+
}
|
|
73
|
+
]
|
|
74
|
+
|
|
64
75
|
def get_open_ai_output_handlers(self):
|
|
65
|
-
return {
|
|
66
|
-
"computer_call" : self.handle_computer_call
|
|
67
|
-
}
|
|
76
|
+
return {"computer_call": self.handle_computer_call}
|
|
68
77
|
|
|
69
78
|
async def handle_computer_call(self, context: ToolContext, **arguments):
|
|
70
|
-
|
|
71
79
|
outputs = await operator.play(computer=self.computer, item=arguments)
|
|
72
80
|
for output in outputs:
|
|
73
81
|
if output["type"] == "computer_call_output":
|
|
74
|
-
if output["output"]
|
|
82
|
+
if output["output"] is not None:
|
|
75
83
|
if output["output"]["type"] == "input_image":
|
|
76
|
-
|
|
77
|
-
b64 : str = output["output"]["image_url"]
|
|
84
|
+
b64: str = output["output"]["image_url"]
|
|
78
85
|
image_data_b64 = b64.split(",", 1)
|
|
79
|
-
|
|
86
|
+
|
|
80
87
|
image_bytes = base64.b64decode(image_data_b64[1])
|
|
81
88
|
render_screen(image_bytes)
|
|
82
|
-
|
|
83
|
-
|
|
89
|
+
|
|
84
90
|
nonlocal computer_toolkit
|
|
85
91
|
if len(computer_toolkit.tools) == 1:
|
|
86
92
|
# HACK: after looking at the page, add the other tools,
|
|
87
93
|
# if we add these first then the computer-use-preview mode fails if it calls them before using the computer
|
|
88
|
-
computer_toolkit.tools.extend(
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
94
|
+
computer_toolkit.tools.extend(
|
|
95
|
+
[
|
|
96
|
+
ScreenshotTool(computer=computer),
|
|
97
|
+
GotoURL(computer=computer),
|
|
98
|
+
]
|
|
99
|
+
)
|
|
92
100
|
return outputs[0]
|
|
93
|
-
|
|
101
|
+
|
|
94
102
|
class ScreenshotTool(Tool):
|
|
95
103
|
def __init__(self, computer: Computer):
|
|
96
104
|
self.computer = computer
|
|
@@ -99,23 +107,20 @@ def make_computer_toolkit(*, operator_cls: Type[Operator], computer_cls: Type[Co
|
|
|
99
107
|
name="screenshot",
|
|
100
108
|
# TODO: give a correct schema
|
|
101
109
|
input_schema={
|
|
102
|
-
"additionalProperties"
|
|
103
|
-
"type"
|
|
104
|
-
"required"
|
|
105
|
-
"properties"
|
|
106
|
-
"full_page" :
|
|
107
|
-
|
|
110
|
+
"additionalProperties": False,
|
|
111
|
+
"type": "object",
|
|
112
|
+
"required": ["full_page", "save_path"],
|
|
113
|
+
"properties": {
|
|
114
|
+
"full_page": {"type": "boolean"},
|
|
115
|
+
"save_path": {
|
|
116
|
+
"type": "string",
|
|
117
|
+
"description": "a file path to save the screenshot to (should end with .png)",
|
|
108
118
|
},
|
|
109
|
-
|
|
110
|
-
"type" : "string",
|
|
111
|
-
"description" : "a file path to save the screenshot to (should end with .png)"
|
|
112
|
-
}
|
|
113
|
-
}
|
|
119
|
+
},
|
|
114
120
|
},
|
|
115
|
-
description="take a screenshot of the current page",
|
|
121
|
+
description="take a screenshot of the current page",
|
|
116
122
|
)
|
|
117
123
|
|
|
118
|
-
|
|
119
124
|
async def execute(self, context: ToolContext, save_path: str, full_page: bool):
|
|
120
125
|
screenshot_bytes = await self.computer.screenshot_bytes(full_page=full_page)
|
|
121
126
|
handle = await context.room.storage.open(path=save_path, overwrite=True)
|
|
@@ -123,7 +128,7 @@ def make_computer_toolkit(*, operator_cls: Type[Operator], computer_cls: Type[Co
|
|
|
123
128
|
await context.room.storage.close(handle=handle)
|
|
124
129
|
|
|
125
130
|
return f"saved screenshot to {save_path}"
|
|
126
|
-
|
|
131
|
+
|
|
127
132
|
class GotoURL(Tool):
|
|
128
133
|
def __init__(self, computer: Computer):
|
|
129
134
|
self.computer = computer
|
|
@@ -133,61 +138,66 @@ def make_computer_toolkit(*, operator_cls: Type[Operator], computer_cls: Type[Co
|
|
|
133
138
|
description="goes to a specific URL. Make sure it starts with http:// or https://",
|
|
134
139
|
# TODO: give a correct schema
|
|
135
140
|
input_schema={
|
|
136
|
-
"additionalProperties"
|
|
137
|
-
"type"
|
|
138
|
-
"required"
|
|
139
|
-
"properties"
|
|
140
|
-
"url"
|
|
141
|
-
"type"
|
|
141
|
+
"additionalProperties": False,
|
|
142
|
+
"type": "object",
|
|
143
|
+
"required": ["url"],
|
|
144
|
+
"properties": {
|
|
145
|
+
"url": {
|
|
146
|
+
"type": "string",
|
|
142
147
|
"description": "Fully qualified URL to navigate to.",
|
|
143
148
|
}
|
|
144
|
-
}
|
|
149
|
+
},
|
|
145
150
|
},
|
|
146
151
|
)
|
|
147
152
|
|
|
148
|
-
|
|
149
153
|
async def execute(self, context: ToolContext, url: str):
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
if url.startswith("https://") == False and url.startswith("http://") == False:
|
|
153
|
-
url = "https://"+url
|
|
154
|
+
if not url.startswith("https://") and not url.startswith("http://"):
|
|
155
|
+
url = "https://" + url
|
|
154
156
|
|
|
155
157
|
await self.computer.goto(url)
|
|
156
158
|
|
|
157
159
|
render_screen(await self.computer.screenshot_bytes(full_page=False))
|
|
158
160
|
|
|
159
|
-
|
|
160
161
|
computer_tool = ComputerTool(computer=computer, operator=operator)
|
|
161
162
|
|
|
162
|
-
computer_toolkit = ComputerToolkit(
|
|
163
|
-
|
|
164
|
-
|
|
163
|
+
computer_toolkit = ComputerToolkit(
|
|
164
|
+
name="meshagent.openai.computer",
|
|
165
|
+
computer=computer,
|
|
166
|
+
operator=operator,
|
|
167
|
+
tools=[computer_tool],
|
|
168
|
+
)
|
|
165
169
|
|
|
166
170
|
return computer_toolkit
|
|
167
171
|
|
|
172
|
+
|
|
168
173
|
async def make_browserbase_toolkit(context: ToolContext, requirement: RequiredToolkit):
|
|
169
|
-
return make_computer_toolkit(
|
|
174
|
+
return make_computer_toolkit(
|
|
175
|
+
operator_cls=Operator(), computer_cls=BrowserbaseBrowser()
|
|
176
|
+
)
|
|
177
|
+
|
|
170
178
|
|
|
171
179
|
register_toolkit_factory("browserbase", make_browserbase_toolkit)
|
|
172
180
|
|
|
173
181
|
|
|
174
182
|
class ComputerAgent(ChatBot):
|
|
175
|
-
def __init__(
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
|
|
179
|
-
|
|
180
|
-
|
|
181
|
-
|
|
182
|
-
|
|
183
|
-
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
|
|
187
|
-
|
|
188
|
-
|
|
183
|
+
def __init__(
|
|
184
|
+
self,
|
|
185
|
+
*,
|
|
186
|
+
name,
|
|
187
|
+
title=None,
|
|
188
|
+
description=None,
|
|
189
|
+
requires=None,
|
|
190
|
+
labels=None,
|
|
191
|
+
computer_cls: Type[Computer] = BrowserbaseBrowser,
|
|
192
|
+
operator_cls: Type[Operator] = Operator,
|
|
193
|
+
rules: Optional[list[str]] = None,
|
|
194
|
+
llm_adapter: Optional[LLMAdapter] = None,
|
|
195
|
+
toolkits: list[Toolkit] = None,
|
|
196
|
+
):
|
|
197
|
+
if rules is None:
|
|
198
|
+
rules = [
|
|
189
199
|
"if asked to go to a URL, you MUST use the goto function to go to the url if it is available",
|
|
190
|
-
"after going directly to a URL, the screen will change so you should take a look at it to know what to do next"
|
|
200
|
+
"after going directly to a URL, the screen will change so you should take a look at it to know what to do next",
|
|
191
201
|
]
|
|
192
202
|
super().__init__(
|
|
193
203
|
name=name,
|
|
@@ -197,36 +207,33 @@ class ComputerAgent(ChatBot):
|
|
|
197
207
|
labels=labels,
|
|
198
208
|
llm_adapter=llm_adapter,
|
|
199
209
|
toolkits=toolkits,
|
|
200
|
-
rules=rules
|
|
210
|
+
rules=rules,
|
|
201
211
|
)
|
|
202
212
|
self.computer_cls = computer_cls
|
|
203
213
|
self.operator_cls = operator_cls
|
|
204
214
|
|
|
205
|
-
async def get_thread_toolkits(
|
|
206
|
-
|
|
207
|
-
|
|
215
|
+
async def get_thread_toolkits(
|
|
216
|
+
self, *, thread_context: ChatThreadContext, participant: RemoteParticipant
|
|
217
|
+
):
|
|
218
|
+
toolkits = await super().get_thread_toolkits(
|
|
219
|
+
thread_context=thread_context, participant=participant
|
|
220
|
+
)
|
|
208
221
|
|
|
209
222
|
def render_screen(image_bytes: bytes):
|
|
210
223
|
for participant in thread_context.participants:
|
|
211
224
|
self.room.messaging.send_message_nowait(
|
|
212
225
|
to=participant,
|
|
213
226
|
type="computer_screen",
|
|
214
|
-
message={
|
|
215
|
-
|
|
216
|
-
attachment=image_bytes
|
|
227
|
+
message={},
|
|
228
|
+
attachment=image_bytes,
|
|
217
229
|
)
|
|
218
|
-
|
|
230
|
+
|
|
219
231
|
computer_toolkit = make_computer_toolkit(
|
|
220
232
|
operator_cls=self.operator_cls,
|
|
221
233
|
computer_cls=self.computer_cls,
|
|
222
|
-
render_screen=render_screen
|
|
234
|
+
render_screen=render_screen,
|
|
223
235
|
)
|
|
224
236
|
|
|
225
237
|
await computer_toolkit.ensure_started()
|
|
226
238
|
|
|
227
|
-
return [
|
|
228
|
-
computer_toolkit,
|
|
229
|
-
*toolkits
|
|
230
|
-
]
|
|
231
|
-
|
|
232
|
-
|
|
239
|
+
return [computer_toolkit, *toolkits]
|
|
@@ -61,7 +61,6 @@ class BasePlaywrightComputer:
|
|
|
61
61
|
|
|
62
62
|
# Set up network interception to flag URLs matching domains in BLOCKED_DOMAINS
|
|
63
63
|
async def handle_route(route: Route, request: Request):
|
|
64
|
-
|
|
65
64
|
url = request.url
|
|
66
65
|
if check_blocklisted_url(url):
|
|
67
66
|
print(f"Flagging blocked domain: {url}")
|
|
@@ -79,18 +78,13 @@ class BasePlaywrightComputer:
|
|
|
79
78
|
if self._playwright:
|
|
80
79
|
await self._context.__aexit__(exc_type, exc_val, exc_tb)
|
|
81
80
|
|
|
82
|
-
def get_current_url(self) -> str:
|
|
83
|
-
if self._page == None:
|
|
84
|
-
return "about:blank"
|
|
85
|
-
|
|
86
81
|
async def ensure_page(self):
|
|
87
82
|
# After a timeout, we might loose our browser
|
|
88
|
-
if self._page
|
|
83
|
+
if self._page is None or not self._browser.is_connected:
|
|
89
84
|
self._browser, self._page = await self._get_browser_and_page()
|
|
90
85
|
|
|
91
86
|
# --- Common "Computer" actions ---
|
|
92
87
|
|
|
93
|
-
|
|
94
88
|
async def screenshot_bytes(self, full_page: bool = False) -> bytes:
|
|
95
89
|
await self.ensure_page()
|
|
96
90
|
png_bytes = await self._page.screenshot(full_page=full_page)
|
|
@@ -146,7 +140,7 @@ class BasePlaywrightComputer:
|
|
|
146
140
|
await self.ensure_page()
|
|
147
141
|
if not path:
|
|
148
142
|
return
|
|
149
|
-
|
|
143
|
+
|
|
150
144
|
await self._page.mouse.move(path[0]["x"], path[0]["y"])
|
|
151
145
|
await self._page.mouse.down()
|
|
152
146
|
for point in path[1:]:
|
|
@@ -1,10 +1,9 @@
|
|
|
1
1
|
import os
|
|
2
|
-
from typing import Tuple
|
|
3
|
-
from playwright.async_api import Browser, Page,
|
|
2
|
+
from typing import Tuple
|
|
3
|
+
from playwright.async_api import Browser, Page, Error as PlaywrightError
|
|
4
4
|
from .base_playwright import BasePlaywrightComputer
|
|
5
5
|
from browserbase import AsyncBrowserbase
|
|
6
6
|
from dotenv import load_dotenv
|
|
7
|
-
import base64
|
|
8
7
|
|
|
9
8
|
load_dotenv()
|
|
10
9
|
|
|
@@ -75,8 +74,7 @@ class BrowserbaseBrowser(BasePlaywrightComputer):
|
|
|
75
74
|
|
|
76
75
|
# Connect to the remote session
|
|
77
76
|
browser = await self._playwright.chromium.connect_over_cdp(
|
|
78
|
-
self.session.connect_url,
|
|
79
|
-
timeout=60000
|
|
77
|
+
self.session.connect_url, timeout=60000
|
|
80
78
|
)
|
|
81
79
|
context = browser.contexts[0]
|
|
82
80
|
|
|
@@ -186,12 +184,13 @@ class BrowserbaseBrowser(BasePlaywrightComputer):
|
|
|
186
184
|
cdp_session = await self._page.context.new_cdp_session(self._page)
|
|
187
185
|
|
|
188
186
|
# Capture screenshot using CDP
|
|
189
|
-
result = await cdp_session.send(
|
|
190
|
-
"format": "png",
|
|
191
|
-
|
|
192
|
-
})
|
|
187
|
+
result = await cdp_session.send(
|
|
188
|
+
"Page.captureScreenshot", {"format": "png", "fromSurface": True}
|
|
189
|
+
)
|
|
193
190
|
|
|
194
|
-
return result[
|
|
191
|
+
return result["data"]
|
|
195
192
|
except PlaywrightError as error:
|
|
196
|
-
print(
|
|
193
|
+
print(
|
|
194
|
+
f"CDP screenshot failed, falling back to standard screenshot: {error}"
|
|
195
|
+
)
|
|
197
196
|
return await super().screenshot()
|
meshagent/computers/computer.py
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
from typing import Protocol, List, Literal, Dict
|
|
2
|
-
|
|
2
|
+
|
|
3
3
|
|
|
4
4
|
class Computer(Protocol):
|
|
5
5
|
"""Defines the 'shape' (methods/properties) our loop expects."""
|
|
@@ -28,9 +28,9 @@ class Computer(Protocol):
|
|
|
28
28
|
async def drag(self, path: List[Dict[str, int]]) -> None: ...
|
|
29
29
|
|
|
30
30
|
async def get_current_url() -> str: ...
|
|
31
|
-
|
|
32
|
-
async def __aenter__(self) ->
|
|
31
|
+
|
|
32
|
+
async def __aenter__(self) -> "Computer":
|
|
33
33
|
return self
|
|
34
34
|
|
|
35
|
-
async def __aexit__(self, exc_type, exc_val, exc_tb) ->
|
|
36
|
-
return self
|
|
35
|
+
async def __aexit__(self, exc_type, exc_val, exc_tb) -> "Computer":
|
|
36
|
+
return self
|
meshagent/computers/docker.py
CHANGED
|
@@ -1,21 +1,20 @@
|
|
|
1
1
|
import subprocess
|
|
2
2
|
import time
|
|
3
|
-
import shlex
|
|
4
3
|
import asyncio
|
|
5
4
|
|
|
6
5
|
|
|
7
6
|
async def _async_check_output(*args, **kwargs):
|
|
8
7
|
proc = await asyncio.create_subprocess_exec(
|
|
9
|
-
*args,
|
|
10
|
-
stdout=asyncio.subprocess.PIPE,
|
|
11
|
-
stderr=asyncio.subprocess.PIPE,
|
|
12
|
-
**kwargs
|
|
8
|
+
*args, stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.PIPE, **kwargs
|
|
13
9
|
)
|
|
14
10
|
stdout, stderr = await proc.communicate()
|
|
15
11
|
if proc.returncode != 0:
|
|
16
|
-
raise subprocess.CalledProcessError(
|
|
12
|
+
raise subprocess.CalledProcessError(
|
|
13
|
+
proc.returncode, args, output=stdout, stderr=stderr
|
|
14
|
+
)
|
|
17
15
|
return stdout
|
|
18
16
|
|
|
17
|
+
|
|
19
18
|
class DockerComputer:
|
|
20
19
|
environment = "linux"
|
|
21
20
|
dimensions = (1280, 720) # Default fallback; will be updated in __enter__.
|
|
@@ -108,8 +107,7 @@ class DockerComputer:
|
|
|
108
107
|
# "base64 /tmp/screenshot.png"
|
|
109
108
|
# )
|
|
110
109
|
cmd = (
|
|
111
|
-
f"export DISPLAY={self.display} && "
|
|
112
|
-
"import -window root png:- | base64 -w 0"
|
|
110
|
+
f"export DISPLAY={self.display} && import -window root png:- | base64 -w 0"
|
|
113
111
|
)
|
|
114
112
|
|
|
115
113
|
return await self._exec(cmd)
|
|
@@ -175,5 +173,7 @@ class DockerComputer:
|
|
|
175
173
|
f"DISPLAY={self.display} xdotool mousemove {start_x} {start_y} mousedown 1"
|
|
176
174
|
)
|
|
177
175
|
for point in path[1:]:
|
|
178
|
-
await self._exec(
|
|
176
|
+
await self._exec(
|
|
177
|
+
f"DISPLAY={self.display} xdotool mousemove {point['x']} {point['y']}"
|
|
178
|
+
)
|
|
179
179
|
await self._exec(f"DISPLAY={self.display} xdotool mouseup 1")
|
|
@@ -11,12 +11,13 @@ class LocalPlaywrightComputer(BasePlaywrightComputer):
|
|
|
11
11
|
|
|
12
12
|
async def _get_browser_and_page(self) -> tuple[Browser, Page]:
|
|
13
13
|
width, height = self.dimensions
|
|
14
|
-
launch_args = [
|
|
14
|
+
launch_args = [
|
|
15
|
+
f"--window-size={width},{height}",
|
|
16
|
+
"--disable-extensions",
|
|
17
|
+
"--disable-file-system",
|
|
18
|
+
]
|
|
15
19
|
browser = await self._playwright.chromium.launch(
|
|
16
|
-
chromium_sandbox=True,
|
|
17
|
-
headless=self.headless,
|
|
18
|
-
args=launch_args,
|
|
19
|
-
env={}
|
|
20
|
+
chromium_sandbox=True, headless=self.headless, args=launch_args, env={}
|
|
20
21
|
)
|
|
21
22
|
page = await browser.new_page()
|
|
22
23
|
await page.set_viewport_size({"width": width, "height": height})
|
meshagent/computers/operator.py
CHANGED
|
@@ -2,12 +2,13 @@ from .computer import Computer
|
|
|
2
2
|
from .utils import check_blocklisted_url
|
|
3
3
|
import json
|
|
4
4
|
|
|
5
|
+
|
|
5
6
|
class Operator:
|
|
6
7
|
def __init__(self):
|
|
7
8
|
self.print_steps = False
|
|
8
9
|
self.show_images = False
|
|
9
10
|
|
|
10
|
-
async def acknowledge_safety_check_callback(self, data: dict):
|
|
11
|
+
async def acknowledge_safety_check_callback(self, data: dict):
|
|
11
12
|
return True
|
|
12
13
|
|
|
13
14
|
async def show_image(self, base_64: str):
|
|
@@ -75,4 +76,4 @@ class Operator:
|
|
|
75
76
|
call_output["output"]["current_url"] = current_url
|
|
76
77
|
|
|
77
78
|
return [call_output]
|
|
78
|
-
return []
|
|
79
|
+
return []
|
meshagent/computers/version.py
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
__version__ = "0.0.
|
|
1
|
+
__version__ = "0.0.38"
|
|
@@ -0,0 +1,65 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: meshagent-computers
|
|
3
|
+
Version: 0.0.38
|
|
4
|
+
Summary: Computer Building Blocks for Meshagent
|
|
5
|
+
License-Expression: Apache-2.0
|
|
6
|
+
Project-URL: Documentation, https://docs.meshagent.com
|
|
7
|
+
Project-URL: Website, https://www.meshagent.com
|
|
8
|
+
Project-URL: Source, https://www.meshagent.com
|
|
9
|
+
Requires-Python: >=3.12
|
|
10
|
+
Description-Content-Type: text/markdown
|
|
11
|
+
License-File: LICENSE
|
|
12
|
+
Requires-Dist: pytest~=8.4
|
|
13
|
+
Requires-Dist: pytest-asyncio~=0.26
|
|
14
|
+
Requires-Dist: openai~=1.86
|
|
15
|
+
Requires-Dist: meshagent-api~=0.0.38
|
|
16
|
+
Requires-Dist: meshagent-agents~=0.0.38
|
|
17
|
+
Requires-Dist: meshagent-tools~=0.0.38
|
|
18
|
+
Requires-Dist: playwright~=1.51
|
|
19
|
+
Requires-Dist: browserbase~=1.2
|
|
20
|
+
Requires-Dist: scrapybara~=2.4
|
|
21
|
+
Dynamic: license-file
|
|
22
|
+
|
|
23
|
+
## MeshAgent Computers
|
|
24
|
+
|
|
25
|
+
The ``meshagent.computers`` package defines abstractions for controlling browsers and operating systems and providing these abilities to agents.
|
|
26
|
+
|
|
27
|
+
### ComputerAgent
|
|
28
|
+
The ComputerAgent in `meshagent-computers` extends the ``ChatBot`` with support for using browsers and computers. The computer agent will periodically send screenshots to participants on the thread using the MeshAgent messaging protocol, by sending a message of the type "computer_screen" and an attachment that contains a binary screenshot.
|
|
29
|
+
|
|
30
|
+
```Python Python
|
|
31
|
+
from meshagent.api import RequiredToolkit
|
|
32
|
+
from meshagent.openai import OpenAIResponsesAdapter
|
|
33
|
+
from meshagent.computers import ComputerAgent, BrowserbaseBrowser, Operator
|
|
34
|
+
from meshagent.api.services import ServiceHost
|
|
35
|
+
|
|
36
|
+
service = ServiceHost()
|
|
37
|
+
|
|
38
|
+
@service.path("/computeragent")
|
|
39
|
+
class BrowserbaseAgent(ComputerAgent):
|
|
40
|
+
def __init__(self):
|
|
41
|
+
super().__init__(
|
|
42
|
+
name="meshagent.browser",
|
|
43
|
+
title="browser agent",
|
|
44
|
+
description="a task runner that can use a browser",
|
|
45
|
+
requires=[RequiredToolkit(name="ui", tools=[])],
|
|
46
|
+
llm_adapter=OpenAIResponsesAdapter(
|
|
47
|
+
model="computer-use-preview",
|
|
48
|
+
response_options={"reasoning": {"generate_summary": "concise"}, "truncation": "auto"},
|
|
49
|
+
),
|
|
50
|
+
labels=["tasks", "computers"],
|
|
51
|
+
computer_cls=BrowserbaseBrowser,
|
|
52
|
+
operator_cls=Operator
|
|
53
|
+
)
|
|
54
|
+
|
|
55
|
+
asyncio.run(service.run())
|
|
56
|
+
```
|
|
57
|
+
|
|
58
|
+
---
|
|
59
|
+
### Learn more about MeshAgent on our website or check out the docs for additional examples!
|
|
60
|
+
|
|
61
|
+
**Website**: [www.meshagent.com](https://www.meshagent.com/)
|
|
62
|
+
|
|
63
|
+
**Documentation**: [docs.meshagent.com](https://docs.meshagent.com/)
|
|
64
|
+
|
|
65
|
+
---
|
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
meshagent/computers/__init__.py,sha256=DEm-FN2iAoC5AJmbIeZR0NrNUDGyh0hWqjdfBhv9GSg,519
|
|
2
|
+
meshagent/computers/agent.py,sha256=FD4lb8ruyTblW8si3Xxwks0d8GofRKse8bSbIEhRM3s,8472
|
|
3
|
+
meshagent/computers/base_playwright.py,sha256=QOzopf6Drs0uP_itg9iusZ2Vn6BvRoV6iQtCRCmNwRA,5887
|
|
4
|
+
meshagent/computers/browserbase.py,sha256=VdvGsasnEMVndi4fSyF5EbXMIRPRMmVzCk603H_dyHQ,8104
|
|
5
|
+
meshagent/computers/computer.py,sha256=znXIMMNxbDBYH0aRfbTjQkVFuAmuS9rqoahBAx_lCrI,1068
|
|
6
|
+
meshagent/computers/docker.py,sha256=ZR9l8HOf9U8FL51oaM9t73emJQbOHjQ7-aC2UzHhDMU,6332
|
|
7
|
+
meshagent/computers/local_playwright.py,sha256=r28blerW1BxiVL0B4Pnkv9Qz5-LtU83rQOrHhXbjeXQ,930
|
|
8
|
+
meshagent/computers/operator.py,sha256=29LDS6Jmg5xxIWD_mpXsA0eP0ZjwNYkwZQFChZzJuNs,2914
|
|
9
|
+
meshagent/computers/scrapybara.py,sha256=IoeV02QcBDkpW05A1NH-lc4jnBEmZ4SPsRo3nnXC3UE,7502
|
|
10
|
+
meshagent/computers/utils.py,sha256=N7Ltjx3HfhJ-CYectNodGsBtddk6mSTHkCH0meqHT70,2080
|
|
11
|
+
meshagent/computers/version.py,sha256=R5QxTjVaID7odO0eBWpOnyCjNQxBZ7cpyruM_NMOoDc,23
|
|
12
|
+
meshagent_computers-0.0.38.dist-info/licenses/LICENSE,sha256=eTt0SPW-sVNdkZe9PS_S8WfCIyLjRXRl7sUBWdlteFg,10254
|
|
13
|
+
meshagent_computers-0.0.38.dist-info/METADATA,sha256=aak3CMOgfzVfpCgFNJiniNzimqqVDGdqi-T3erO17tU,2427
|
|
14
|
+
meshagent_computers-0.0.38.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
|
|
15
|
+
meshagent_computers-0.0.38.dist-info/top_level.txt,sha256=GlcXnHtRP6m7zlG3Df04M35OsHtNXy_DY09oFwWrH74,10
|
|
16
|
+
meshagent_computers-0.0.38.dist-info/RECORD,,
|
|
@@ -1,23 +0,0 @@
|
|
|
1
|
-
Metadata-Version: 2.4
|
|
2
|
-
Name: meshagent-computers
|
|
3
|
-
Version: 0.0.37
|
|
4
|
-
Summary: Computer Building Blocks for Meshagent
|
|
5
|
-
License-Expression: Apache-2.0
|
|
6
|
-
Project-URL: Documentation, https://docs.meshagent.com
|
|
7
|
-
Project-URL: Website, https://www.meshagent.com
|
|
8
|
-
Project-URL: Source, https://www.meshagent.com
|
|
9
|
-
Requires-Python: >=3.12
|
|
10
|
-
Description-Content-Type: text/markdown
|
|
11
|
-
License-File: LICENSE
|
|
12
|
-
Requires-Dist: pytest~=8.3
|
|
13
|
-
Requires-Dist: pytest-asyncio~=0.26
|
|
14
|
-
Requires-Dist: openai~=1.86
|
|
15
|
-
Requires-Dist: meshagent-api~=0.0.37
|
|
16
|
-
Requires-Dist: meshagent-agents~=0.0.37
|
|
17
|
-
Requires-Dist: meshagent-tools~=0.0.37
|
|
18
|
-
Requires-Dist: playwright~=1.51
|
|
19
|
-
Requires-Dist: browserbase~=1.2
|
|
20
|
-
Requires-Dist: scrapybara~=2.4
|
|
21
|
-
Dynamic: license-file
|
|
22
|
-
|
|
23
|
-
### Meshagent Computers
|
|
@@ -1,16 +0,0 @@
|
|
|
1
|
-
meshagent/computers/__init__.py,sha256=PEzgkWE_sB6TrBA8crwWM7QaNPYWJisCKT81tk6pepQ,321
|
|
2
|
-
meshagent/computers/agent.py,sha256=B24gs5-IW26SNu5K5MqVWYMuuCFzTOu1aeGxgcO9wmE,8700
|
|
3
|
-
meshagent/computers/base_playwright.py,sha256=NhQoOzWmekBmobQ2kvN7w4I3hZeyMIuMKI2eHbG1_mk,6017
|
|
4
|
-
meshagent/computers/browserbase.py,sha256=ZT-ZDndkBD-MHD3UjxXpwpx7-_VCkkanjW1rIJr7-98,8161
|
|
5
|
-
meshagent/computers/computer.py,sha256=465GKDV3cJgzp1SB5B0xzLoftMKU0AxAuBVrZ9drBBc,1072
|
|
6
|
-
meshagent/computers/docker.py,sha256=BGkrlspIYlg3WViaqctyRUye-fONfF40AvLXOQuXPQ8,6331
|
|
7
|
-
meshagent/computers/local_playwright.py,sha256=W7VhxIYEXn2BO3wZZVsc-hH-CU7NTssxFGO7pwz_n1I,919
|
|
8
|
-
meshagent/computers/operator.py,sha256=JTyGvoPnKOqUDaZv58p_2n5Q6m6dm8V214VMIhVEu08,2913
|
|
9
|
-
meshagent/computers/scrapybara.py,sha256=IoeV02QcBDkpW05A1NH-lc4jnBEmZ4SPsRo3nnXC3UE,7502
|
|
10
|
-
meshagent/computers/utils.py,sha256=N7Ltjx3HfhJ-CYectNodGsBtddk6mSTHkCH0meqHT70,2080
|
|
11
|
-
meshagent/computers/version.py,sha256=JaGEpJ5xP3R4j7pGgCziGajlIRjy1_NJdv_OaXPQius,22
|
|
12
|
-
meshagent_computers-0.0.37.dist-info/licenses/LICENSE,sha256=eTt0SPW-sVNdkZe9PS_S8WfCIyLjRXRl7sUBWdlteFg,10254
|
|
13
|
-
meshagent_computers-0.0.37.dist-info/METADATA,sha256=xb_U3QnUlcaDHsxRrCK40KavlFCueHLXWPpJlSJl2qE,728
|
|
14
|
-
meshagent_computers-0.0.37.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
|
|
15
|
-
meshagent_computers-0.0.37.dist-info/top_level.txt,sha256=GlcXnHtRP6m7zlG3Df04M35OsHtNXy_DY09oFwWrH74,10
|
|
16
|
-
meshagent_computers-0.0.37.dist-info/RECORD,,
|
|
File without changes
|
{meshagent_computers-0.0.37.dist-info → meshagent_computers-0.0.38.dist-info}/licenses/LICENSE
RENAMED
|
File without changes
|
|
File without changes
|