fleet-python 0.2.16__py3-none-any.whl → 0.2.18__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of fleet-python might be problematic. Click here for more details.
- fleet/__init__.py +0 -5
- fleet/_async/client.py +1 -1
- fleet/client.py +5 -11
- fleet/verifiers/code.py +2 -1
- {fleet_python-0.2.16.dist-info → fleet_python-0.2.18.dist-info}/METADATA +1 -1
- {fleet_python-0.2.16.dist-info → fleet_python-0.2.18.dist-info}/RECORD +9 -11
- fleet/_async/playwright.py +0 -291
- fleet/playwright.py +0 -290
- {fleet_python-0.2.16.dist-info → fleet_python-0.2.18.dist-info}/WHEEL +0 -0
- {fleet_python-0.2.16.dist-info → fleet_python-0.2.18.dist-info}/licenses/LICENSE +0 -0
- {fleet_python-0.2.16.dist-info → fleet_python-0.2.18.dist-info}/top_level.txt +0 -0
fleet/__init__.py
CHANGED
|
@@ -37,9 +37,6 @@ from .verifiers import (
|
|
|
37
37
|
TASK_SUCCESSFUL_SCORE,
|
|
38
38
|
)
|
|
39
39
|
|
|
40
|
-
# Import Playwright wrapper
|
|
41
|
-
from .playwright import FleetPlaywrightWrapper
|
|
42
|
-
|
|
43
40
|
# Import async verifiers (default verifier is async for modern usage)
|
|
44
41
|
from ._async.verifiers import (
|
|
45
42
|
verifier,
|
|
@@ -75,8 +72,6 @@ __all__ = [
|
|
|
75
72
|
"FleetAPIError",
|
|
76
73
|
"FleetTimeoutError",
|
|
77
74
|
"FleetConfigurationError",
|
|
78
|
-
# Playwright wrapper
|
|
79
|
-
"FleetPlaywrightWrapper",
|
|
80
75
|
# Verifiers (async is default)
|
|
81
76
|
"verifier",
|
|
82
77
|
"verifier_sync",
|
fleet/_async/client.py
CHANGED
|
@@ -70,7 +70,7 @@ class AsyncEnv(EnvironmentBase):
|
|
|
70
70
|
# Remove the current app name (e.g., /sentry) to get the root
|
|
71
71
|
if '/' in base_url:
|
|
72
72
|
parts = base_url.rsplit('/', 1)
|
|
73
|
-
if len(parts) == 2:
|
|
73
|
+
if len(parts) == 2 and parts[0] != "https:/":
|
|
74
74
|
base_url = parts[0]
|
|
75
75
|
|
|
76
76
|
self._apps[name] = AsyncInstanceClient(
|
fleet/client.py
CHANGED
|
@@ -164,7 +164,7 @@ class Environment(EnvironmentBase):
|
|
|
164
164
|
# Remove the current app name (e.g., /sentry) to get the root
|
|
165
165
|
if '/' in base_url:
|
|
166
166
|
parts = base_url.rsplit('/', 1)
|
|
167
|
-
if len(parts) == 2:
|
|
167
|
+
if len(parts) == 2 and parts[0] != "https:/":
|
|
168
168
|
base_url = parts[0]
|
|
169
169
|
|
|
170
170
|
self._apps[name] = InstanceClient(
|
|
@@ -537,12 +537,6 @@ class Fleet:
|
|
|
537
537
|
browser = new_env.browser()
|
|
538
538
|
browser.start(width=snapshot.viewport_size[0], height=snapshot.viewport_size[1])
|
|
539
539
|
|
|
540
|
-
from fleet.playwright import FleetPlaywrightWrapper
|
|
541
|
-
|
|
542
|
-
playwright_wrapper = FleetPlaywrightWrapper(
|
|
543
|
-
cdp_url=browser.cdp_url(), instance_client=new_env.instance
|
|
544
|
-
)
|
|
545
|
-
|
|
546
540
|
# Replay tool logs in order
|
|
547
541
|
validation_errors = []
|
|
548
542
|
last_timestamp = None
|
|
@@ -559,7 +553,7 @@ class Fleet:
|
|
|
559
553
|
|
|
560
554
|
# Replay the tool action
|
|
561
555
|
_replay_tool_action(
|
|
562
|
-
|
|
556
|
+
None,
|
|
563
557
|
tool_log,
|
|
564
558
|
new_env.instance._client,
|
|
565
559
|
replay_session_id,
|
|
@@ -588,7 +582,7 @@ class Fleet:
|
|
|
588
582
|
|
|
589
583
|
if validate:
|
|
590
584
|
validation = _validate_resumed_state(
|
|
591
|
-
new_env, snapshot,
|
|
585
|
+
new_env, snapshot, None, validation_errors
|
|
592
586
|
)
|
|
593
587
|
|
|
594
588
|
return new_env, validation
|
|
@@ -664,7 +658,7 @@ def _execute_verifier_remote(
|
|
|
664
658
|
|
|
665
659
|
|
|
666
660
|
def _replay_tool_action(
|
|
667
|
-
playwright_wrapper
|
|
661
|
+
playwright_wrapper,
|
|
668
662
|
tool_log: ToolLogEntry,
|
|
669
663
|
client: "SyncWrapper",
|
|
670
664
|
session_id: str,
|
|
@@ -760,7 +754,7 @@ def _replay_tool_action(
|
|
|
760
754
|
def _validate_resumed_state(
|
|
761
755
|
new_env: Environment,
|
|
762
756
|
snapshot: EnvironmentSnapshot,
|
|
763
|
-
playwright_wrapper
|
|
757
|
+
playwright_wrapper,
|
|
764
758
|
existing_errors: List[str],
|
|
765
759
|
) -> SnapshotValidation:
|
|
766
760
|
"""Validate that the resumed state matches the snapshot."""
|
fleet/verifiers/code.py
CHANGED
|
@@ -1 +1,2 @@
|
|
|
1
|
-
TASK_SUCCESSFUL_SCORE = 1
|
|
1
|
+
TASK_SUCCESSFUL_SCORE = 1
|
|
2
|
+
TASK_FAILED_SCORE = 0
|
|
@@ -15,20 +15,18 @@ examples/openai_example.py,sha256=I2vk_SJN9BkSRQCYRJfbtGJ-HJ2xzQj-lOjwqmLos5M,82
|
|
|
15
15
|
examples/openai_simple_example.py,sha256=I42ytIwv0INgDO39pp1MOQSqsJz2YYH8GeNNBaUtq3A,1748
|
|
16
16
|
examples/query_builder_example.py,sha256=Q3lUBETHpu1aS2FXAO79ADYqCxOjMMMZNgCcFVapiII,3918
|
|
17
17
|
examples/quickstart.py,sha256=1VT39IRRhemsJgxi0O0gprdpcw7HB4pYO97GAYagIcg,3788
|
|
18
|
-
fleet/__init__.py,sha256=
|
|
18
|
+
fleet/__init__.py,sha256=9EsG068VCTqCCyBA2jw8IMLw4_oFkWLhtY394CtjEgM,2234
|
|
19
19
|
fleet/base.py,sha256=0yYuMN0lBkrfTTZBt5NQp5112xWgziuWEk4GuHJB1wE,9189
|
|
20
|
-
fleet/client.py,sha256=
|
|
20
|
+
fleet/client.py,sha256=HrHkxQMk9_2GsyG7_edauKets3xySglCUuXW3WtXmTE,28263
|
|
21
21
|
fleet/config.py,sha256=zd19st83NJdW9DdOq7Irpc0x-iUnMad0JOtAr_nD5DM,273
|
|
22
22
|
fleet/exceptions.py,sha256=fUmPwWhnT8SR97lYsRq0kLHQHKtSh2eJS0VQ2caSzEI,5055
|
|
23
23
|
fleet/models.py,sha256=YMHFAgBF4OqoAOv-arHze3bPDDZ5DAzb3CXhVLF6pPw,9019
|
|
24
|
-
fleet/playwright.py,sha256=BmRvez5DUa0ttAQB084hPAyt9_8WxdzCGBGF-GZbTuQ,8593
|
|
25
24
|
fleet/tasks.py,sha256=w-0vVGfEuCWRHMEJ73SN141J7Lz2pD_0v-nNG4TyJTU,1645
|
|
26
25
|
fleet/types.py,sha256=eXeI8BFmiU5hln0PVvJbUZs7BSjl6wSqAtN9zaJT6yY,652
|
|
27
26
|
fleet/_async/__init__.py,sha256=AJWCnuo7XKja4yBb8fK2wX7ntciLXQrpzdRHwjTRP6M,62
|
|
28
27
|
fleet/_async/base.py,sha256=s0rYOtXsMJeitOvpa-Oh8ciLV226p_TIPp3fplzWvV4,9209
|
|
29
|
-
fleet/_async/client.py,sha256=
|
|
28
|
+
fleet/_async/client.py,sha256=yyNawgGro2sfFh0ok7DtbjnyxaBGrHg9RtTFEISvwdE,10581
|
|
30
29
|
fleet/_async/exceptions.py,sha256=fUmPwWhnT8SR97lYsRq0kLHQHKtSh2eJS0VQ2caSzEI,5055
|
|
31
|
-
fleet/_async/playwright.py,sha256=NXKcwJezl6jEV2CKBgeaB7zLshFWypgSA5GYXevDRX8,8908
|
|
32
30
|
fleet/_async/tasks.py,sha256=w-0vVGfEuCWRHMEJ73SN141J7Lz2pD_0v-nNG4TyJTU,1645
|
|
33
31
|
fleet/_async/env/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
34
32
|
fleet/_async/env/client.py,sha256=PQGKcvnVTr3o2KNsgQM7ZBqaCjimv-wEn-XiYGm44Ko,753
|
|
@@ -55,15 +53,15 @@ fleet/resources/mcp.py,sha256=55LBwhNDqSeFVCD0O3OP9m2DdJE-KEYRQiCnVyze8DM,1864
|
|
|
55
53
|
fleet/resources/sqlite.py,sha256=U0KneMzFKFrcuXcGrl4gkFPNoWST7_gWINphcA_UPJA,26251
|
|
56
54
|
fleet/verifiers/__init__.py,sha256=-dm2x0wC8UbfGcMCbPtjb0-LospGUhD-S3Pm4oha6BY,445
|
|
57
55
|
fleet/verifiers/bundler.py,sha256=A4yR3wBOcVZYFAv87CD58QlJn6L4QXeilrasnVm8n74,26185
|
|
58
|
-
fleet/verifiers/code.py,sha256=
|
|
56
|
+
fleet/verifiers/code.py,sha256=EOi6ES8Zdzlm9iybRFaJmz9t2W4Ulo2wrCdbEBqxzbc,47
|
|
59
57
|
fleet/verifiers/db.py,sha256=tssmvJjDHuBIy8qlL_P5-UdmEFUw2DZcqLsWZ8ot3Xw,27766
|
|
60
58
|
fleet/verifiers/decorator.py,sha256=Q-KHhicnIYFwX7FX_VZguzNfu8ZslqNUeWxcS2CwNVY,3386
|
|
61
59
|
fleet/verifiers/sql_differ.py,sha256=dmiGCFXVMEMbAX519OjhVqgA8ZvhnvdmC1BVpL7QCF0,6490
|
|
62
60
|
fleet/verifiers/verifier.py,sha256=C5L24M7VBHWKaWlryC1EdW1g7keZBGzVSjgXC_SlpXA,12205
|
|
63
|
-
fleet_python-0.2.
|
|
61
|
+
fleet_python-0.2.18.dist-info/licenses/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
|
|
64
62
|
scripts/fix_sync_imports.py,sha256=BIQfnaOoQ7bwR1c-pDqH9ifN47W1bwL7OafWVXZNkuA,4368
|
|
65
63
|
scripts/unasync.py,sha256=--Fmaae47o-dZ1HYgX1c3Nvi-rMjcFymTRlJcWWnmpw,725
|
|
66
|
-
fleet_python-0.2.
|
|
67
|
-
fleet_python-0.2.
|
|
68
|
-
fleet_python-0.2.
|
|
69
|
-
fleet_python-0.2.
|
|
64
|
+
fleet_python-0.2.18.dist-info/METADATA,sha256=aRYxF2vM2DDfTbvSrAjiCmArnhmriLOpOAqEI5IgQO8,3297
|
|
65
|
+
fleet_python-0.2.18.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
|
|
66
|
+
fleet_python-0.2.18.dist-info/top_level.txt,sha256=_3DSmTohvSDf3AIP_BYfGzhwO1ECFwuzg83X-wHCx3Y,23
|
|
67
|
+
fleet_python-0.2.18.dist-info/RECORD,,
|
fleet/_async/playwright.py
DELETED
|
@@ -1,291 +0,0 @@
|
|
|
1
|
-
import base64
|
|
2
|
-
from typing import List, Dict, Any
|
|
3
|
-
from playwright.async_api import async_playwright, Browser, Page
|
|
4
|
-
from .client import AsyncEnv
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
# Key mapping for computer use actions
|
|
8
|
-
CUA_KEY_TO_PLAYWRIGHT_KEY = {
|
|
9
|
-
"/": "Divide",
|
|
10
|
-
"\\": "Backslash",
|
|
11
|
-
"alt": "Alt",
|
|
12
|
-
"arrowdown": "ArrowDown",
|
|
13
|
-
"arrowleft": "ArrowLeft",
|
|
14
|
-
"arrowright": "ArrowRight",
|
|
15
|
-
"arrowup": "ArrowUp",
|
|
16
|
-
"backspace": "Backspace",
|
|
17
|
-
"capslock": "CapsLock",
|
|
18
|
-
"cmd": "Meta",
|
|
19
|
-
"ctrl": "Control",
|
|
20
|
-
"delete": "Delete",
|
|
21
|
-
"end": "End",
|
|
22
|
-
"enter": "Enter",
|
|
23
|
-
"esc": "Escape",
|
|
24
|
-
"home": "Home",
|
|
25
|
-
"insert": "Insert",
|
|
26
|
-
"option": "Alt",
|
|
27
|
-
"pagedown": "PageDown",
|
|
28
|
-
"pageup": "PageUp",
|
|
29
|
-
"shift": "Shift",
|
|
30
|
-
"space": " ",
|
|
31
|
-
"super": "Meta",
|
|
32
|
-
"tab": "Tab",
|
|
33
|
-
"win": "Meta",
|
|
34
|
-
}
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
class AsyncFleetPlaywrightWrapper:
|
|
38
|
-
"""
|
|
39
|
-
A wrapper that adds Playwright browser automation to Fleet environment instances.
|
|
40
|
-
|
|
41
|
-
This class handles:
|
|
42
|
-
- Browser connection via CDP
|
|
43
|
-
- Computer actions (click, scroll, type, etc.)
|
|
44
|
-
- Screenshot capture
|
|
45
|
-
- Integration with OpenAI computer use API
|
|
46
|
-
|
|
47
|
-
Usage:
|
|
48
|
-
instance = await fleet.env.make(env_key="hubspot", version="v1.2.7")
|
|
49
|
-
browser = AsyncFleetPlaywrightWrapper(instance)
|
|
50
|
-
await browser.start()
|
|
51
|
-
|
|
52
|
-
# Use browser methods
|
|
53
|
-
screenshot = await browser.screenshot()
|
|
54
|
-
tools = [browser.openai_cua_tool]
|
|
55
|
-
|
|
56
|
-
# Clean up when done
|
|
57
|
-
await browser.close()
|
|
58
|
-
"""
|
|
59
|
-
|
|
60
|
-
def get_environment(self):
|
|
61
|
-
return "browser"
|
|
62
|
-
|
|
63
|
-
def get_dimensions(self):
|
|
64
|
-
return (1920, 1080)
|
|
65
|
-
|
|
66
|
-
def __init__(
|
|
67
|
-
self,
|
|
68
|
-
env: AsyncEnv,
|
|
69
|
-
display_width: int = 1920,
|
|
70
|
-
display_height: int = 1080,
|
|
71
|
-
):
|
|
72
|
-
"""
|
|
73
|
-
Initialize the Fleet Playwright wrapper.
|
|
74
|
-
|
|
75
|
-
Args:
|
|
76
|
-
env: Fleet environment instance
|
|
77
|
-
display_width: Browser viewport width
|
|
78
|
-
display_height: Browser viewport height
|
|
79
|
-
"""
|
|
80
|
-
self.env = env
|
|
81
|
-
self.display_width = display_width
|
|
82
|
-
self.display_height = display_height
|
|
83
|
-
|
|
84
|
-
self._playwright = None
|
|
85
|
-
self._browser: Browser | None = None
|
|
86
|
-
self._page: Page | None = None
|
|
87
|
-
self._started = False
|
|
88
|
-
|
|
89
|
-
async def start(self):
|
|
90
|
-
"""Start the browser and establish connection."""
|
|
91
|
-
if self._started:
|
|
92
|
-
return
|
|
93
|
-
|
|
94
|
-
# Start Playwright
|
|
95
|
-
self._playwright = await async_playwright().start()
|
|
96
|
-
|
|
97
|
-
# Start browser on the Fleet instance
|
|
98
|
-
print("Starting browser...")
|
|
99
|
-
await self.env.browser().start()
|
|
100
|
-
cdp = await self.env.browser().describe()
|
|
101
|
-
|
|
102
|
-
# Connect to browser
|
|
103
|
-
self._browser = await self._playwright.chromium.connect_over_cdp(
|
|
104
|
-
cdp.cdp_browser_url
|
|
105
|
-
)
|
|
106
|
-
self._page = self._browser.contexts[0].pages[0]
|
|
107
|
-
await self._page.set_viewport_size(
|
|
108
|
-
{"width": self.display_width, "height": self.display_height}
|
|
109
|
-
)
|
|
110
|
-
|
|
111
|
-
self._started = True
|
|
112
|
-
print(f"Track agent: {cdp.cdp_devtools_url}")
|
|
113
|
-
|
|
114
|
-
async def close(self):
|
|
115
|
-
"""Close the browser connection."""
|
|
116
|
-
if self._playwright:
|
|
117
|
-
await self._playwright.stop()
|
|
118
|
-
self._playwright = None
|
|
119
|
-
self._browser = None
|
|
120
|
-
self._page = None
|
|
121
|
-
self._started = False
|
|
122
|
-
|
|
123
|
-
def _ensure_started(self):
|
|
124
|
-
"""Ensure browser is started before operations."""
|
|
125
|
-
if not self._started:
|
|
126
|
-
raise RuntimeError("Browser not started. Call await browser.start() first.")
|
|
127
|
-
|
|
128
|
-
@property
|
|
129
|
-
def openai_cua_tool(self) -> Dict[str, Any]:
|
|
130
|
-
"""
|
|
131
|
-
Tool definition for OpenAI computer use API.
|
|
132
|
-
|
|
133
|
-
Returns:
|
|
134
|
-
Tool definition dict for use with OpenAI responses API
|
|
135
|
-
"""
|
|
136
|
-
return {
|
|
137
|
-
"type": "computer_use_preview",
|
|
138
|
-
"display_width": self.display_width,
|
|
139
|
-
"display_height": self.display_height,
|
|
140
|
-
"environment": "browser",
|
|
141
|
-
}
|
|
142
|
-
|
|
143
|
-
async def screenshot(self) -> str:
|
|
144
|
-
"""
|
|
145
|
-
Take a screenshot and return base64 encoded string.
|
|
146
|
-
|
|
147
|
-
Returns:
|
|
148
|
-
Base64 encoded PNG screenshot
|
|
149
|
-
"""
|
|
150
|
-
self._ensure_started()
|
|
151
|
-
|
|
152
|
-
png_bytes = await self._page.screenshot(full_page=False)
|
|
153
|
-
return base64.b64encode(png_bytes).decode("utf-8")
|
|
154
|
-
|
|
155
|
-
def get_current_url(self) -> str:
|
|
156
|
-
"""Get the current page URL."""
|
|
157
|
-
self._ensure_started()
|
|
158
|
-
return self._page.url
|
|
159
|
-
|
|
160
|
-
async def execute_computer_action(self, action: Dict[str, Any]) -> Dict[str, Any]:
|
|
161
|
-
"""
|
|
162
|
-
Execute a computer action and return the result for OpenAI API.
|
|
163
|
-
|
|
164
|
-
Args:
|
|
165
|
-
action: Computer action dict from OpenAI response
|
|
166
|
-
|
|
167
|
-
Returns:
|
|
168
|
-
Result dict for computer_call_output
|
|
169
|
-
"""
|
|
170
|
-
self._ensure_started()
|
|
171
|
-
|
|
172
|
-
action_type = action["type"]
|
|
173
|
-
action_args = {k: v for k, v in action.items() if k != "type"}
|
|
174
|
-
|
|
175
|
-
print(f"Executing: {action_type}({action_args})")
|
|
176
|
-
|
|
177
|
-
# Execute the action
|
|
178
|
-
if hasattr(self, f"_{action_type}"):
|
|
179
|
-
method = getattr(self, f"_{action_type}")
|
|
180
|
-
await method(**action_args)
|
|
181
|
-
else:
|
|
182
|
-
raise ValueError(f"Unsupported action type: {action_type}")
|
|
183
|
-
|
|
184
|
-
# Take screenshot after action
|
|
185
|
-
screenshot_base64 = await self.screenshot()
|
|
186
|
-
|
|
187
|
-
return {
|
|
188
|
-
"type": "input_image",
|
|
189
|
-
"image_url": f"data:image/png;base64,{screenshot_base64}",
|
|
190
|
-
"current_url": self.get_current_url(),
|
|
191
|
-
}
|
|
192
|
-
|
|
193
|
-
# Computer action implementations
|
|
194
|
-
async def _click(self, x: int, y: int, button: str = "left") -> None:
|
|
195
|
-
"""Click at coordinates."""
|
|
196
|
-
self._ensure_started()
|
|
197
|
-
await self._page.mouse.click(x, y, button=button)
|
|
198
|
-
|
|
199
|
-
async def _double_click(self, x: int, y: int) -> None:
|
|
200
|
-
"""Double-click at coordinates."""
|
|
201
|
-
self._ensure_started()
|
|
202
|
-
await self._page.mouse.dblclick(x, y)
|
|
203
|
-
|
|
204
|
-
async def _scroll(self, x: int, y: int, scroll_x: int, scroll_y: int) -> None:
|
|
205
|
-
"""Scroll from coordinates."""
|
|
206
|
-
self._ensure_started()
|
|
207
|
-
await self._page.mouse.move(x, y)
|
|
208
|
-
await self._page.evaluate(f"window.scrollBy({scroll_x}, {scroll_y})")
|
|
209
|
-
|
|
210
|
-
async def _type(self, text: str) -> None:
|
|
211
|
-
"""Type text."""
|
|
212
|
-
self._ensure_started()
|
|
213
|
-
await self._page.keyboard.type(text)
|
|
214
|
-
|
|
215
|
-
async def _keypress(self, keys: List[str]) -> None:
|
|
216
|
-
"""Press key combination."""
|
|
217
|
-
self._ensure_started()
|
|
218
|
-
mapped_keys = [CUA_KEY_TO_PLAYWRIGHT_KEY.get(key.lower(), key) for key in keys]
|
|
219
|
-
for key in mapped_keys:
|
|
220
|
-
await self._page.keyboard.down(key)
|
|
221
|
-
for key in reversed(mapped_keys):
|
|
222
|
-
await self._page.keyboard.up(key)
|
|
223
|
-
|
|
224
|
-
async def _move(self, x: int, y: int) -> None:
|
|
225
|
-
"""Move mouse to coordinates."""
|
|
226
|
-
self._ensure_started()
|
|
227
|
-
await self._page.mouse.move(x, y)
|
|
228
|
-
|
|
229
|
-
async def _drag(self, path: List[Dict[str, int]]) -> None:
|
|
230
|
-
"""Drag mouse along path."""
|
|
231
|
-
self._ensure_started()
|
|
232
|
-
if not path:
|
|
233
|
-
return
|
|
234
|
-
await self._page.mouse.move(path[0]["x"], path[0]["y"])
|
|
235
|
-
await self._page.mouse.down()
|
|
236
|
-
for point in path[1:]:
|
|
237
|
-
await self._page.mouse.move(point["x"], point["y"])
|
|
238
|
-
await self._page.mouse.up()
|
|
239
|
-
|
|
240
|
-
async def _wait(self, ms: int = 1000) -> None:
|
|
241
|
-
"""Wait for specified milliseconds."""
|
|
242
|
-
import asyncio
|
|
243
|
-
|
|
244
|
-
await asyncio.sleep(ms / 1000)
|
|
245
|
-
|
|
246
|
-
# Browser-specific actions
|
|
247
|
-
async def _goto(self, url: str) -> None:
|
|
248
|
-
"""Navigate to URL."""
|
|
249
|
-
self._ensure_started()
|
|
250
|
-
try:
|
|
251
|
-
await self._page.goto(url)
|
|
252
|
-
except Exception as e:
|
|
253
|
-
print(f"Error navigating to {url}: {e}")
|
|
254
|
-
|
|
255
|
-
async def _back(self) -> None:
|
|
256
|
-
"""Go back in browser history."""
|
|
257
|
-
self._ensure_started()
|
|
258
|
-
await self._page.go_back()
|
|
259
|
-
|
|
260
|
-
async def _forward(self) -> None:
|
|
261
|
-
"""Go forward in browser history."""
|
|
262
|
-
self._ensure_started()
|
|
263
|
-
await self._page.go_forward()
|
|
264
|
-
|
|
265
|
-
async def _refresh(self) -> None:
|
|
266
|
-
"""Refresh the page."""
|
|
267
|
-
self._ensure_started()
|
|
268
|
-
await self._page.reload()
|
|
269
|
-
|
|
270
|
-
# ------------------------------------------------------------------
|
|
271
|
-
# Public aliases (no leading underscore) expected by the Agent &
|
|
272
|
-
# OpenAI computer-use API. They forward directly to the underscored
|
|
273
|
-
# implementations above so the external interface matches the older
|
|
274
|
-
# BasePlaywrightComputer class.
|
|
275
|
-
# ------------------------------------------------------------------
|
|
276
|
-
|
|
277
|
-
# Mouse / keyboard actions
|
|
278
|
-
click = _click
|
|
279
|
-
double_click = _double_click
|
|
280
|
-
scroll = _scroll
|
|
281
|
-
type = _type # noqa: A003 – shadowing built-in for API compatibility
|
|
282
|
-
keypress = _keypress
|
|
283
|
-
move = _move
|
|
284
|
-
drag = _drag
|
|
285
|
-
wait = _wait
|
|
286
|
-
|
|
287
|
-
# Browser navigation actions
|
|
288
|
-
goto = _goto
|
|
289
|
-
back = _back
|
|
290
|
-
forward = _forward
|
|
291
|
-
refresh = _refresh
|
fleet/playwright.py
DELETED
|
@@ -1,290 +0,0 @@
|
|
|
1
|
-
import base64
|
|
2
|
-
from typing import List, Dict, Any
|
|
3
|
-
from playwright.sync_api import sync_playwright, Browser, Page
|
|
4
|
-
from .client import Environment
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
# Key mapping for computer use actions
|
|
8
|
-
CUA_KEY_TO_PLAYWRIGHT_KEY = {
|
|
9
|
-
"/": "Divide",
|
|
10
|
-
"\\": "Backslash",
|
|
11
|
-
"alt": "Alt",
|
|
12
|
-
"arrowdown": "ArrowDown",
|
|
13
|
-
"arrowleft": "ArrowLeft",
|
|
14
|
-
"arrowright": "ArrowRight",
|
|
15
|
-
"arrowup": "ArrowUp",
|
|
16
|
-
"backspace": "Backspace",
|
|
17
|
-
"capslock": "CapsLock",
|
|
18
|
-
"cmd": "Meta",
|
|
19
|
-
"ctrl": "Control",
|
|
20
|
-
"delete": "Delete",
|
|
21
|
-
"end": "End",
|
|
22
|
-
"enter": "Enter",
|
|
23
|
-
"esc": "Escape",
|
|
24
|
-
"home": "Home",
|
|
25
|
-
"insert": "Insert",
|
|
26
|
-
"option": "Alt",
|
|
27
|
-
"pagedown": "PageDown",
|
|
28
|
-
"pageup": "PageUp",
|
|
29
|
-
"shift": "Shift",
|
|
30
|
-
"space": " ",
|
|
31
|
-
"super": "Meta",
|
|
32
|
-
"tab": "Tab",
|
|
33
|
-
"win": "Meta",
|
|
34
|
-
}
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
class FleetPlaywrightWrapper:
|
|
38
|
-
"""
|
|
39
|
-
A wrapper that adds Playwright browser automation to Fleet environment instances.
|
|
40
|
-
|
|
41
|
-
This class handles:
|
|
42
|
-
- Browser connection via CDP
|
|
43
|
-
- Computer actions (click, scroll, type, etc.)
|
|
44
|
-
- Screenshot capture
|
|
45
|
-
- Integration with OpenAI computer use API
|
|
46
|
-
|
|
47
|
-
Usage:
|
|
48
|
-
instance = fleet.env.make(env_key="hubspot", version="v1.2.7")
|
|
49
|
-
browser = FleetPlaywrightWrapper(instance)
|
|
50
|
-
browser.start()
|
|
51
|
-
|
|
52
|
-
# Use browser methods
|
|
53
|
-
screenshot = browser.screenshot()
|
|
54
|
-
tools = [browser.openai_cua_tool]
|
|
55
|
-
|
|
56
|
-
# Clean up when done
|
|
57
|
-
browser.close()
|
|
58
|
-
"""
|
|
59
|
-
|
|
60
|
-
def get_environment(self):
|
|
61
|
-
return "browser"
|
|
62
|
-
|
|
63
|
-
def get_dimensions(self):
|
|
64
|
-
return (1920, 1080)
|
|
65
|
-
|
|
66
|
-
def __init__(
|
|
67
|
-
self,
|
|
68
|
-
env: Environment,
|
|
69
|
-
display_width: int = 1920,
|
|
70
|
-
display_height: int = 1080,
|
|
71
|
-
):
|
|
72
|
-
"""
|
|
73
|
-
Initialize the Fleet Playwright wrapper.
|
|
74
|
-
|
|
75
|
-
Args:
|
|
76
|
-
env: Fleet environment instance
|
|
77
|
-
display_width: Browser viewport width
|
|
78
|
-
display_height: Browser viewport height
|
|
79
|
-
"""
|
|
80
|
-
self.env = env
|
|
81
|
-
self.display_width = display_width
|
|
82
|
-
self.display_height = display_height
|
|
83
|
-
|
|
84
|
-
self._playwright = None
|
|
85
|
-
self._browser: Browser | None = None
|
|
86
|
-
self._page: Page | None = None
|
|
87
|
-
self._started = False
|
|
88
|
-
|
|
89
|
-
def start(self):
|
|
90
|
-
"""Start the browser and establish connection."""
|
|
91
|
-
if self._started:
|
|
92
|
-
return
|
|
93
|
-
|
|
94
|
-
# Start Playwright
|
|
95
|
-
self._playwright = sync_playwright().start()
|
|
96
|
-
|
|
97
|
-
# Start browser on the Fleet instance
|
|
98
|
-
print("Starting browser...")
|
|
99
|
-
self.env.browser().start()
|
|
100
|
-
cdp = self.env.browser().describe()
|
|
101
|
-
|
|
102
|
-
# Connect to browser
|
|
103
|
-
self._browser = self._playwright.chromium.connect_over_cdp(
|
|
104
|
-
cdp.cdp_browser_url
|
|
105
|
-
)
|
|
106
|
-
self._page = self._browser.contexts[0].pages[0]
|
|
107
|
-
self._page.set_viewport_size(
|
|
108
|
-
{"width": self.display_width, "height": self.display_height}
|
|
109
|
-
)
|
|
110
|
-
|
|
111
|
-
self._started = True
|
|
112
|
-
print(f"Track agent: {cdp.cdp_devtools_url}")
|
|
113
|
-
|
|
114
|
-
def close(self):
|
|
115
|
-
"""Close the browser connection."""
|
|
116
|
-
if self._playwright:
|
|
117
|
-
self._playwright.stop()
|
|
118
|
-
self._playwright = None
|
|
119
|
-
self._browser = None
|
|
120
|
-
self._page = None
|
|
121
|
-
self._started = False
|
|
122
|
-
|
|
123
|
-
def _ensure_started(self):
|
|
124
|
-
"""Ensure browser is started before operations."""
|
|
125
|
-
if not self._started:
|
|
126
|
-
raise RuntimeError("Browser not started. Call browser.start() first.")
|
|
127
|
-
|
|
128
|
-
@property
|
|
129
|
-
def openai_cua_tool(self) -> Dict[str, Any]:
|
|
130
|
-
"""
|
|
131
|
-
Tool definition for OpenAI computer use API.
|
|
132
|
-
|
|
133
|
-
Returns:
|
|
134
|
-
Tool definition dict for use with OpenAI responses API
|
|
135
|
-
"""
|
|
136
|
-
return {
|
|
137
|
-
"type": "computer_use_preview",
|
|
138
|
-
"display_width": self.display_width,
|
|
139
|
-
"display_height": self.display_height,
|
|
140
|
-
"environment": "browser",
|
|
141
|
-
}
|
|
142
|
-
|
|
143
|
-
def screenshot(self) -> str:
|
|
144
|
-
"""
|
|
145
|
-
Take a screenshot and return base64 encoded string.
|
|
146
|
-
|
|
147
|
-
Returns:
|
|
148
|
-
Base64 encoded PNG screenshot
|
|
149
|
-
"""
|
|
150
|
-
self._ensure_started()
|
|
151
|
-
|
|
152
|
-
png_bytes = self._page.screenshot(full_page=False)
|
|
153
|
-
return base64.b64encode(png_bytes).decode("utf-8")
|
|
154
|
-
|
|
155
|
-
def get_current_url(self) -> str:
|
|
156
|
-
"""Get the current page URL."""
|
|
157
|
-
self._ensure_started()
|
|
158
|
-
return self._page.url
|
|
159
|
-
|
|
160
|
-
def execute_computer_action(self, action: Dict[str, Any]) -> Dict[str, Any]:
|
|
161
|
-
"""
|
|
162
|
-
Execute a computer action and return the result for OpenAI API.
|
|
163
|
-
|
|
164
|
-
Args:
|
|
165
|
-
action: Computer action dict from OpenAI response
|
|
166
|
-
|
|
167
|
-
Returns:
|
|
168
|
-
Result dict for computer_call_output
|
|
169
|
-
"""
|
|
170
|
-
self._ensure_started()
|
|
171
|
-
|
|
172
|
-
action_type = action["type"]
|
|
173
|
-
action_args = {k: v for k, v in action.items() if k != "type"}
|
|
174
|
-
|
|
175
|
-
print(f"Executing: {action_type}({action_args})")
|
|
176
|
-
|
|
177
|
-
# Execute the action
|
|
178
|
-
if hasattr(self, f"_{action_type}"):
|
|
179
|
-
method = getattr(self, f"_{action_type}")
|
|
180
|
-
method(**action_args)
|
|
181
|
-
else:
|
|
182
|
-
raise ValueError(f"Unsupported action type: {action_type}")
|
|
183
|
-
|
|
184
|
-
# Take screenshot after action
|
|
185
|
-
screenshot_base64 = self.screenshot()
|
|
186
|
-
|
|
187
|
-
return {
|
|
188
|
-
"type": "input_image",
|
|
189
|
-
"image_url": f"data:image/png;base64,{screenshot_base64}",
|
|
190
|
-
"current_url": self.get_current_url(),
|
|
191
|
-
}
|
|
192
|
-
|
|
193
|
-
# Computer action implementations
|
|
194
|
-
def _click(self, x: int, y: int, button: str = "left") -> None:
|
|
195
|
-
"""Click at coordinates."""
|
|
196
|
-
self._ensure_started()
|
|
197
|
-
self._page.mouse.click(x, y, button=button)
|
|
198
|
-
|
|
199
|
-
def _double_click(self, x: int, y: int) -> None:
|
|
200
|
-
"""Double-click at coordinates."""
|
|
201
|
-
self._ensure_started()
|
|
202
|
-
self._page.mouse.dblclick(x, y)
|
|
203
|
-
|
|
204
|
-
def _scroll(self, x: int, y: int, scroll_x: int, scroll_y: int) -> None:
|
|
205
|
-
"""Scroll from coordinates."""
|
|
206
|
-
self._ensure_started()
|
|
207
|
-
self._page.mouse.move(x, y)
|
|
208
|
-
self._page.evaluate(f"window.scrollBy({scroll_x}, {scroll_y})")
|
|
209
|
-
|
|
210
|
-
def _type(self, text: str) -> None:
|
|
211
|
-
"""Type text."""
|
|
212
|
-
self._ensure_started()
|
|
213
|
-
self._page.keyboard.type(text)
|
|
214
|
-
|
|
215
|
-
def _keypress(self, keys: List[str]) -> None:
|
|
216
|
-
"""Press key combination."""
|
|
217
|
-
self._ensure_started()
|
|
218
|
-
mapped_keys = [CUA_KEY_TO_PLAYWRIGHT_KEY.get(key.lower(), key) for key in keys]
|
|
219
|
-
for key in mapped_keys:
|
|
220
|
-
self._page.keyboard.down(key)
|
|
221
|
-
for key in reversed(mapped_keys):
|
|
222
|
-
self._page.keyboard.up(key)
|
|
223
|
-
|
|
224
|
-
def _move(self, x: int, y: int) -> None:
|
|
225
|
-
"""Move mouse to coordinates."""
|
|
226
|
-
self._ensure_started()
|
|
227
|
-
self._page.mouse.move(x, y)
|
|
228
|
-
|
|
229
|
-
def _drag(self, path: List[Dict[str, int]]) -> None:
|
|
230
|
-
"""Drag mouse along path."""
|
|
231
|
-
self._ensure_started()
|
|
232
|
-
if not path:
|
|
233
|
-
return
|
|
234
|
-
self._page.mouse.move(path[0]["x"], path[0]["y"])
|
|
235
|
-
self._page.mouse.down()
|
|
236
|
-
for point in path[1:]:
|
|
237
|
-
self._page.mouse.move(point["x"], point["y"])
|
|
238
|
-
self._page.mouse.up()
|
|
239
|
-
|
|
240
|
-
def _wait(self, ms: int = 1000) -> None:
|
|
241
|
-
"""Wait for specified milliseconds."""
|
|
242
|
-
|
|
243
|
-
time.sleep(ms / 1000)
|
|
244
|
-
|
|
245
|
-
# Browser-specific actions
|
|
246
|
-
def _goto(self, url: str) -> None:
|
|
247
|
-
"""Navigate to URL."""
|
|
248
|
-
self._ensure_started()
|
|
249
|
-
try:
|
|
250
|
-
self._page.goto(url)
|
|
251
|
-
except Exception as e:
|
|
252
|
-
print(f"Error navigating to {url}: {e}")
|
|
253
|
-
|
|
254
|
-
def _back(self) -> None:
|
|
255
|
-
"""Go back in browser history."""
|
|
256
|
-
self._ensure_started()
|
|
257
|
-
self._page.go_back()
|
|
258
|
-
|
|
259
|
-
def _forward(self) -> None:
|
|
260
|
-
"""Go forward in browser history."""
|
|
261
|
-
self._ensure_started()
|
|
262
|
-
self._page.go_forward()
|
|
263
|
-
|
|
264
|
-
def _refresh(self) -> None:
|
|
265
|
-
"""Refresh the page."""
|
|
266
|
-
self._ensure_started()
|
|
267
|
-
self._page.reload()
|
|
268
|
-
|
|
269
|
-
# ------------------------------------------------------------------
|
|
270
|
-
# Public aliases (no leading underscore) expected by the Agent &
|
|
271
|
-
# OpenAI computer-use API. They forward directly to the underscored
|
|
272
|
-
# implementations above so the external interface matches the older
|
|
273
|
-
# BasePlaywrightComputer class.
|
|
274
|
-
# ------------------------------------------------------------------
|
|
275
|
-
|
|
276
|
-
# Mouse / keyboard actions
|
|
277
|
-
click = _click
|
|
278
|
-
double_click = _double_click
|
|
279
|
-
scroll = _scroll
|
|
280
|
-
type = _type # noqa: A003 – shadowing built-in for API compatibility
|
|
281
|
-
keypress = _keypress
|
|
282
|
-
move = _move
|
|
283
|
-
drag = _drag
|
|
284
|
-
wait = _wait
|
|
285
|
-
|
|
286
|
-
# Browser navigation actions
|
|
287
|
-
goto = _goto
|
|
288
|
-
back = _back
|
|
289
|
-
forward = _forward
|
|
290
|
-
refresh = _refresh
|
|
File without changes
|
|
File without changes
|
|
File without changes
|