optexity-browser-use 0.9.5__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- browser_use/__init__.py +157 -0
- browser_use/actor/__init__.py +11 -0
- browser_use/actor/element.py +1175 -0
- browser_use/actor/mouse.py +134 -0
- browser_use/actor/page.py +561 -0
- browser_use/actor/playground/flights.py +41 -0
- browser_use/actor/playground/mixed_automation.py +54 -0
- browser_use/actor/playground/playground.py +236 -0
- browser_use/actor/utils.py +176 -0
- browser_use/agent/cloud_events.py +282 -0
- browser_use/agent/gif.py +424 -0
- browser_use/agent/judge.py +170 -0
- browser_use/agent/message_manager/service.py +473 -0
- browser_use/agent/message_manager/utils.py +52 -0
- browser_use/agent/message_manager/views.py +98 -0
- browser_use/agent/prompts.py +413 -0
- browser_use/agent/service.py +2316 -0
- browser_use/agent/system_prompt.md +185 -0
- browser_use/agent/system_prompt_flash.md +10 -0
- browser_use/agent/system_prompt_no_thinking.md +183 -0
- browser_use/agent/views.py +743 -0
- browser_use/browser/__init__.py +41 -0
- browser_use/browser/cloud/cloud.py +203 -0
- browser_use/browser/cloud/views.py +89 -0
- browser_use/browser/events.py +578 -0
- browser_use/browser/profile.py +1158 -0
- browser_use/browser/python_highlights.py +548 -0
- browser_use/browser/session.py +3225 -0
- browser_use/browser/session_manager.py +399 -0
- browser_use/browser/video_recorder.py +162 -0
- browser_use/browser/views.py +200 -0
- browser_use/browser/watchdog_base.py +260 -0
- browser_use/browser/watchdogs/__init__.py +0 -0
- browser_use/browser/watchdogs/aboutblank_watchdog.py +253 -0
- browser_use/browser/watchdogs/crash_watchdog.py +335 -0
- browser_use/browser/watchdogs/default_action_watchdog.py +2729 -0
- browser_use/browser/watchdogs/dom_watchdog.py +817 -0
- browser_use/browser/watchdogs/downloads_watchdog.py +1277 -0
- browser_use/browser/watchdogs/local_browser_watchdog.py +461 -0
- browser_use/browser/watchdogs/permissions_watchdog.py +43 -0
- browser_use/browser/watchdogs/popups_watchdog.py +143 -0
- browser_use/browser/watchdogs/recording_watchdog.py +126 -0
- browser_use/browser/watchdogs/screenshot_watchdog.py +62 -0
- browser_use/browser/watchdogs/security_watchdog.py +280 -0
- browser_use/browser/watchdogs/storage_state_watchdog.py +335 -0
- browser_use/cli.py +2359 -0
- browser_use/code_use/__init__.py +16 -0
- browser_use/code_use/formatting.py +192 -0
- browser_use/code_use/namespace.py +665 -0
- browser_use/code_use/notebook_export.py +276 -0
- browser_use/code_use/service.py +1340 -0
- browser_use/code_use/system_prompt.md +574 -0
- browser_use/code_use/utils.py +150 -0
- browser_use/code_use/views.py +171 -0
- browser_use/config.py +505 -0
- browser_use/controller/__init__.py +3 -0
- browser_use/dom/enhanced_snapshot.py +161 -0
- browser_use/dom/markdown_extractor.py +169 -0
- browser_use/dom/playground/extraction.py +312 -0
- browser_use/dom/playground/multi_act.py +32 -0
- browser_use/dom/serializer/clickable_elements.py +200 -0
- browser_use/dom/serializer/code_use_serializer.py +287 -0
- browser_use/dom/serializer/eval_serializer.py +478 -0
- browser_use/dom/serializer/html_serializer.py +212 -0
- browser_use/dom/serializer/paint_order.py +197 -0
- browser_use/dom/serializer/serializer.py +1170 -0
- browser_use/dom/service.py +825 -0
- browser_use/dom/utils.py +129 -0
- browser_use/dom/views.py +906 -0
- browser_use/exceptions.py +5 -0
- browser_use/filesystem/__init__.py +0 -0
- browser_use/filesystem/file_system.py +619 -0
- browser_use/init_cmd.py +376 -0
- browser_use/integrations/gmail/__init__.py +24 -0
- browser_use/integrations/gmail/actions.py +115 -0
- browser_use/integrations/gmail/service.py +225 -0
- browser_use/llm/__init__.py +155 -0
- browser_use/llm/anthropic/chat.py +242 -0
- browser_use/llm/anthropic/serializer.py +312 -0
- browser_use/llm/aws/__init__.py +36 -0
- browser_use/llm/aws/chat_anthropic.py +242 -0
- browser_use/llm/aws/chat_bedrock.py +289 -0
- browser_use/llm/aws/serializer.py +257 -0
- browser_use/llm/azure/chat.py +91 -0
- browser_use/llm/base.py +57 -0
- browser_use/llm/browser_use/__init__.py +3 -0
- browser_use/llm/browser_use/chat.py +201 -0
- browser_use/llm/cerebras/chat.py +193 -0
- browser_use/llm/cerebras/serializer.py +109 -0
- browser_use/llm/deepseek/chat.py +212 -0
- browser_use/llm/deepseek/serializer.py +109 -0
- browser_use/llm/exceptions.py +29 -0
- browser_use/llm/google/__init__.py +3 -0
- browser_use/llm/google/chat.py +542 -0
- browser_use/llm/google/serializer.py +120 -0
- browser_use/llm/groq/chat.py +229 -0
- browser_use/llm/groq/parser.py +158 -0
- browser_use/llm/groq/serializer.py +159 -0
- browser_use/llm/messages.py +238 -0
- browser_use/llm/models.py +271 -0
- browser_use/llm/oci_raw/__init__.py +10 -0
- browser_use/llm/oci_raw/chat.py +443 -0
- browser_use/llm/oci_raw/serializer.py +229 -0
- browser_use/llm/ollama/chat.py +97 -0
- browser_use/llm/ollama/serializer.py +143 -0
- browser_use/llm/openai/chat.py +264 -0
- browser_use/llm/openai/like.py +15 -0
- browser_use/llm/openai/serializer.py +165 -0
- browser_use/llm/openrouter/chat.py +211 -0
- browser_use/llm/openrouter/serializer.py +26 -0
- browser_use/llm/schema.py +176 -0
- browser_use/llm/views.py +48 -0
- browser_use/logging_config.py +330 -0
- browser_use/mcp/__init__.py +18 -0
- browser_use/mcp/__main__.py +12 -0
- browser_use/mcp/client.py +544 -0
- browser_use/mcp/controller.py +264 -0
- browser_use/mcp/server.py +1114 -0
- browser_use/observability.py +204 -0
- browser_use/py.typed +0 -0
- browser_use/sandbox/__init__.py +41 -0
- browser_use/sandbox/sandbox.py +637 -0
- browser_use/sandbox/views.py +132 -0
- browser_use/screenshots/__init__.py +1 -0
- browser_use/screenshots/service.py +52 -0
- browser_use/sync/__init__.py +6 -0
- browser_use/sync/auth.py +357 -0
- browser_use/sync/service.py +161 -0
- browser_use/telemetry/__init__.py +51 -0
- browser_use/telemetry/service.py +112 -0
- browser_use/telemetry/views.py +101 -0
- browser_use/tokens/__init__.py +0 -0
- browser_use/tokens/custom_pricing.py +24 -0
- browser_use/tokens/mappings.py +4 -0
- browser_use/tokens/service.py +580 -0
- browser_use/tokens/views.py +108 -0
- browser_use/tools/registry/service.py +572 -0
- browser_use/tools/registry/views.py +174 -0
- browser_use/tools/service.py +1675 -0
- browser_use/tools/utils.py +82 -0
- browser_use/tools/views.py +100 -0
- browser_use/utils.py +670 -0
- optexity_browser_use-0.9.5.dist-info/METADATA +344 -0
- optexity_browser_use-0.9.5.dist-info/RECORD +147 -0
- optexity_browser_use-0.9.5.dist-info/WHEEL +4 -0
- optexity_browser_use-0.9.5.dist-info/entry_points.txt +3 -0
- optexity_browser_use-0.9.5.dist-info/licenses/LICENSE +21 -0
|
@@ -0,0 +1,3225 @@
|
|
|
1
|
+
"""Event-driven browser session with backwards compatibility."""
|
|
2
|
+
|
|
3
|
+
import asyncio
|
|
4
|
+
import logging
|
|
5
|
+
from functools import cached_property
|
|
6
|
+
from pathlib import Path
|
|
7
|
+
from typing import TYPE_CHECKING, Any, Literal, Self, Union, cast, overload
|
|
8
|
+
from uuid import UUID
|
|
9
|
+
|
|
10
|
+
import httpx
|
|
11
|
+
from bubus import EventBus
|
|
12
|
+
from cdp_use import CDPClient
|
|
13
|
+
from cdp_use.cdp.fetch import AuthRequiredEvent, RequestPausedEvent
|
|
14
|
+
from cdp_use.cdp.network import Cookie
|
|
15
|
+
from cdp_use.cdp.target import AttachedToTargetEvent, SessionID, TargetID
|
|
16
|
+
from pydantic import BaseModel, ConfigDict, Field, PrivateAttr
|
|
17
|
+
from uuid_extensions import uuid7str
|
|
18
|
+
|
|
19
|
+
from browser_use.browser.cloud.cloud import CloudBrowserAuthError, CloudBrowserClient, CloudBrowserError
|
|
20
|
+
|
|
21
|
+
# CDP logging is now handled by setup_logging() in logging_config.py
|
|
22
|
+
# It automatically sets CDP logs to the same level as browser_use logs
|
|
23
|
+
from browser_use.browser.cloud.views import CloudBrowserParams, CreateBrowserRequest, ProxyCountryCode
|
|
24
|
+
from browser_use.browser.events import (
|
|
25
|
+
AgentFocusChangedEvent,
|
|
26
|
+
BrowserConnectedEvent,
|
|
27
|
+
BrowserErrorEvent,
|
|
28
|
+
BrowserLaunchEvent,
|
|
29
|
+
BrowserLaunchResult,
|
|
30
|
+
BrowserStartEvent,
|
|
31
|
+
BrowserStateRequestEvent,
|
|
32
|
+
BrowserStopEvent,
|
|
33
|
+
BrowserStoppedEvent,
|
|
34
|
+
CloseTabEvent,
|
|
35
|
+
FileDownloadedEvent,
|
|
36
|
+
NavigateToUrlEvent,
|
|
37
|
+
NavigationCompleteEvent,
|
|
38
|
+
NavigationStartedEvent,
|
|
39
|
+
SwitchTabEvent,
|
|
40
|
+
TabClosedEvent,
|
|
41
|
+
TabCreatedEvent,
|
|
42
|
+
)
|
|
43
|
+
from browser_use.browser.profile import BrowserProfile, ProxySettings
|
|
44
|
+
from browser_use.browser.views import BrowserStateSummary, TabInfo
|
|
45
|
+
from browser_use.dom.views import DOMRect, EnhancedDOMTreeNode, TargetInfo
|
|
46
|
+
from browser_use.observability import observe_debug
|
|
47
|
+
from browser_use.utils import _log_pretty_url, is_new_tab_page
|
|
48
|
+
|
|
49
|
+
if TYPE_CHECKING:
|
|
50
|
+
from browser_use.actor.page import Page
|
|
51
|
+
|
|
52
|
+
DEFAULT_BROWSER_PROFILE = BrowserProfile()
|
|
53
|
+
|
|
54
|
+
_LOGGED_UNIQUE_SESSION_IDS = set() # track unique session IDs that have been logged to make sure we always assign a unique enough id to new sessions and avoid ambiguity in logs
|
|
55
|
+
red = '\033[91m'
|
|
56
|
+
reset = '\033[0m'
|
|
57
|
+
|
|
58
|
+
|
|
59
|
+
class CDPSession(BaseModel):
|
|
60
|
+
"""Info about a single CDP session bound to a specific target.
|
|
61
|
+
|
|
62
|
+
Can optionally use its own WebSocket connection for better isolation.
|
|
63
|
+
"""
|
|
64
|
+
|
|
65
|
+
model_config = ConfigDict(arbitrary_types_allowed=True, revalidate_instances='never')
|
|
66
|
+
|
|
67
|
+
cdp_client: CDPClient
|
|
68
|
+
|
|
69
|
+
target_id: TargetID
|
|
70
|
+
session_id: SessionID
|
|
71
|
+
title: str = 'Unknown title'
|
|
72
|
+
url: str = 'about:blank'
|
|
73
|
+
|
|
74
|
+
@classmethod
|
|
75
|
+
async def for_target(
|
|
76
|
+
cls,
|
|
77
|
+
cdp_client: CDPClient,
|
|
78
|
+
target_id: TargetID,
|
|
79
|
+
domains: list[str] | None = None,
|
|
80
|
+
):
|
|
81
|
+
"""Create a CDP session for a target using the shared WebSocket.
|
|
82
|
+
|
|
83
|
+
Args:
|
|
84
|
+
cdp_client: The shared CDP client (root WebSocket connection)
|
|
85
|
+
target_id: Target ID to attach to
|
|
86
|
+
domains: List of CDP domains to enable. If None, enables default domains.
|
|
87
|
+
"""
|
|
88
|
+
# Always use shared CDP client (event-driven approach)
|
|
89
|
+
cdp_session = cls(
|
|
90
|
+
cdp_client=cdp_client,
|
|
91
|
+
target_id=target_id,
|
|
92
|
+
session_id='connecting',
|
|
93
|
+
)
|
|
94
|
+
return await cdp_session.attach(domains=domains)
|
|
95
|
+
|
|
96
|
+
async def attach(self, domains: list[str] | None = None) -> Self:
|
|
97
|
+
result = await self.cdp_client.send.Target.attachToTarget(
|
|
98
|
+
params={
|
|
99
|
+
'targetId': self.target_id,
|
|
100
|
+
'flatten': True, # removed filter as a param because it doesn't exist at https://chromedevtools.github.io/devtools-protocol/tot/Target/#method-attachToTarget
|
|
101
|
+
}
|
|
102
|
+
)
|
|
103
|
+
self.session_id = result['sessionId']
|
|
104
|
+
|
|
105
|
+
# Use specified domains or default domains
|
|
106
|
+
domains = domains or ['Page', 'DOM', 'DOMSnapshot', 'Accessibility', 'Runtime', 'Inspector']
|
|
107
|
+
|
|
108
|
+
# Enable all domains in parallel
|
|
109
|
+
enable_tasks = []
|
|
110
|
+
for domain in domains:
|
|
111
|
+
# Get the enable method, e.g. self.cdp_client.send.Page.enable(session_id=self.session_id)
|
|
112
|
+
domain_api = getattr(self.cdp_client.send, domain, None)
|
|
113
|
+
# Browser and Target domains don't use session_id, dont pass it for those
|
|
114
|
+
enable_kwargs = {} if domain in ['Browser', 'Target'] else {'session_id': self.session_id}
|
|
115
|
+
assert domain_api and hasattr(domain_api, 'enable'), (
|
|
116
|
+
f'{domain_api} is not a recognized CDP domain with a .enable() method'
|
|
117
|
+
)
|
|
118
|
+
enable_tasks.append(domain_api.enable(**enable_kwargs))
|
|
119
|
+
|
|
120
|
+
results = await asyncio.gather(*enable_tasks, return_exceptions=True)
|
|
121
|
+
if any(isinstance(result, Exception) for result in results):
|
|
122
|
+
raise RuntimeError(f'Failed to enable requested CDP domain: {results}')
|
|
123
|
+
|
|
124
|
+
# in case 'Debugger' domain is enabled, disable breakpoints on the page so it doesnt pause on crashes / debugger statements
|
|
125
|
+
# also covered by Runtime.runIfWaitingForDebugger() calls in get_or_create_cdp_session()
|
|
126
|
+
try:
|
|
127
|
+
await self.cdp_client.send.Debugger.setSkipAllPauses(params={'skip': True}, session_id=self.session_id)
|
|
128
|
+
# if 'Debugger' not in domains:
|
|
129
|
+
# await self.cdp_client.send.Debugger.disable()
|
|
130
|
+
# await cdp_session.cdp_client.send.EventBreakpoints.disable(session_id=cdp_session.session_id)
|
|
131
|
+
except Exception:
|
|
132
|
+
# self.logger.warning(f'Failed to disable page JS breakpoints: {e}')
|
|
133
|
+
pass
|
|
134
|
+
|
|
135
|
+
target_info = await self.get_target_info()
|
|
136
|
+
self.title = target_info['title']
|
|
137
|
+
self.url = target_info['url']
|
|
138
|
+
return self
|
|
139
|
+
|
|
140
|
+
async def disconnect(self) -> None:
|
|
141
|
+
"""Disconnect session (no-op since we use shared WebSocket)."""
|
|
142
|
+
# With event-driven approach, all sessions share the root WebSocket
|
|
143
|
+
# Nothing to disconnect - only the root client is disconnected on browser.stop()
|
|
144
|
+
pass
|
|
145
|
+
|
|
146
|
+
async def get_tab_info(self) -> TabInfo:
|
|
147
|
+
target_info = await self.get_target_info()
|
|
148
|
+
return TabInfo(
|
|
149
|
+
target_id=target_info['targetId'],
|
|
150
|
+
url=target_info['url'],
|
|
151
|
+
title=target_info['title'],
|
|
152
|
+
)
|
|
153
|
+
|
|
154
|
+
async def get_target_info(self) -> TargetInfo:
|
|
155
|
+
result = await self.cdp_client.send.Target.getTargetInfo(params={'targetId': self.target_id})
|
|
156
|
+
return result['targetInfo']
|
|
157
|
+
|
|
158
|
+
|
|
159
|
+
class BrowserSession(BaseModel):
|
|
160
|
+
"""Event-driven browser session with backwards compatibility.
|
|
161
|
+
|
|
162
|
+
This class provides a 2-layer architecture:
|
|
163
|
+
- High-level event handling for agents/tools
|
|
164
|
+
- Direct CDP/Playwright calls for browser operations
|
|
165
|
+
|
|
166
|
+
Supports both event-driven and imperative calling styles.
|
|
167
|
+
|
|
168
|
+
Browser configuration is stored in the browser_profile, session identity in direct fields:
|
|
169
|
+
```python
|
|
170
|
+
# Direct settings (recommended for most users)
|
|
171
|
+
session = BrowserSession(headless=True, user_data_dir='./profile')
|
|
172
|
+
|
|
173
|
+
# Or use a profile (for advanced use cases)
|
|
174
|
+
session = BrowserSession(browser_profile=BrowserProfile(...))
|
|
175
|
+
|
|
176
|
+
# Access session fields directly, browser settings via profile or property
|
|
177
|
+
print(session.id) # Session field
|
|
178
|
+
```
|
|
179
|
+
"""
|
|
180
|
+
|
|
181
|
+
model_config = ConfigDict(
|
|
182
|
+
arbitrary_types_allowed=True,
|
|
183
|
+
validate_assignment=True,
|
|
184
|
+
extra='forbid',
|
|
185
|
+
revalidate_instances='never', # resets private attrs on every model rebuild
|
|
186
|
+
)
|
|
187
|
+
|
|
188
|
+
# Overload 1: Cloud browser mode (use cloud-specific params)
|
|
189
|
+
@overload
|
|
190
|
+
def __init__(
|
|
191
|
+
self,
|
|
192
|
+
*,
|
|
193
|
+
# Cloud browser params - use these for cloud mode
|
|
194
|
+
cloud_profile_id: UUID | str | None = None,
|
|
195
|
+
cloud_proxy_country_code: ProxyCountryCode | None = None,
|
|
196
|
+
cloud_timeout: int | None = None,
|
|
197
|
+
# Backward compatibility aliases
|
|
198
|
+
profile_id: UUID | str | None = None,
|
|
199
|
+
proxy_country_code: ProxyCountryCode | None = None,
|
|
200
|
+
timeout: int | None = None,
|
|
201
|
+
use_cloud: bool | None = None,
|
|
202
|
+
cloud_browser: bool | None = None, # Backward compatibility alias
|
|
203
|
+
cloud_browser_params: CloudBrowserParams | None = None,
|
|
204
|
+
# Common params that work with cloud
|
|
205
|
+
id: str | None = None,
|
|
206
|
+
headers: dict[str, str] | None = None,
|
|
207
|
+
allowed_domains: list[str] | None = None,
|
|
208
|
+
keep_alive: bool | None = None,
|
|
209
|
+
minimum_wait_page_load_time: float | None = None,
|
|
210
|
+
wait_for_network_idle_page_load_time: float | None = None,
|
|
211
|
+
wait_between_actions: float | None = None,
|
|
212
|
+
auto_download_pdfs: bool | None = None,
|
|
213
|
+
cookie_whitelist_domains: list[str] | None = None,
|
|
214
|
+
cross_origin_iframes: bool | None = None,
|
|
215
|
+
highlight_elements: bool | None = None,
|
|
216
|
+
dom_highlight_elements: bool | None = None,
|
|
217
|
+
paint_order_filtering: bool | None = None,
|
|
218
|
+
max_iframes: int | None = None,
|
|
219
|
+
max_iframe_depth: int | None = None,
|
|
220
|
+
) -> None: ...
|
|
221
|
+
|
|
222
|
+
# Overload 2: Local browser mode (use local browser params)
|
|
223
|
+
@overload
|
|
224
|
+
def __init__(
|
|
225
|
+
self,
|
|
226
|
+
*,
|
|
227
|
+
# Core configuration for local
|
|
228
|
+
id: str | None = None,
|
|
229
|
+
cdp_url: str | None = None,
|
|
230
|
+
browser_profile: BrowserProfile | None = None,
|
|
231
|
+
# Local browser launch params
|
|
232
|
+
executable_path: str | Path | None = None,
|
|
233
|
+
headless: bool | None = None,
|
|
234
|
+
user_data_dir: str | Path | None = None,
|
|
235
|
+
args: list[str] | None = None,
|
|
236
|
+
downloads_path: str | Path | None = None,
|
|
237
|
+
# Common params
|
|
238
|
+
headers: dict[str, str] | None = None,
|
|
239
|
+
allowed_domains: list[str] | None = None,
|
|
240
|
+
keep_alive: bool | None = None,
|
|
241
|
+
minimum_wait_page_load_time: float | None = None,
|
|
242
|
+
wait_for_network_idle_page_load_time: float | None = None,
|
|
243
|
+
wait_between_actions: float | None = None,
|
|
244
|
+
auto_download_pdfs: bool | None = None,
|
|
245
|
+
cookie_whitelist_domains: list[str] | None = None,
|
|
246
|
+
cross_origin_iframes: bool | None = None,
|
|
247
|
+
highlight_elements: bool | None = None,
|
|
248
|
+
dom_highlight_elements: bool | None = None,
|
|
249
|
+
paint_order_filtering: bool | None = None,
|
|
250
|
+
max_iframes: int | None = None,
|
|
251
|
+
max_iframe_depth: int | None = None,
|
|
252
|
+
# All other local params
|
|
253
|
+
env: dict[str, str | float | bool] | None = None,
|
|
254
|
+
ignore_default_args: list[str] | Literal[True] | None = None,
|
|
255
|
+
channel: str | None = None,
|
|
256
|
+
chromium_sandbox: bool | None = None,
|
|
257
|
+
devtools: bool | None = None,
|
|
258
|
+
traces_dir: str | Path | None = None,
|
|
259
|
+
accept_downloads: bool | None = None,
|
|
260
|
+
permissions: list[str] | None = None,
|
|
261
|
+
user_agent: str | None = None,
|
|
262
|
+
screen: dict | None = None,
|
|
263
|
+
viewport: dict | None = None,
|
|
264
|
+
no_viewport: bool | None = None,
|
|
265
|
+
device_scale_factor: float | None = None,
|
|
266
|
+
record_har_content: str | None = None,
|
|
267
|
+
record_har_mode: str | None = None,
|
|
268
|
+
record_har_path: str | Path | None = None,
|
|
269
|
+
record_video_dir: str | Path | None = None,
|
|
270
|
+
record_video_framerate: int | None = None,
|
|
271
|
+
record_video_size: dict | None = None,
|
|
272
|
+
storage_state: str | Path | dict[str, Any] | None = None,
|
|
273
|
+
disable_security: bool | None = None,
|
|
274
|
+
deterministic_rendering: bool | None = None,
|
|
275
|
+
proxy: ProxySettings | None = None,
|
|
276
|
+
enable_default_extensions: bool | None = None,
|
|
277
|
+
window_size: dict | None = None,
|
|
278
|
+
window_position: dict | None = None,
|
|
279
|
+
filter_highlight_ids: bool | None = None,
|
|
280
|
+
profile_directory: str | None = None,
|
|
281
|
+
) -> None: ...
|
|
282
|
+
|
|
283
|
+
def __init__(
|
|
284
|
+
self,
|
|
285
|
+
# Core configuration
|
|
286
|
+
id: str | None = None,
|
|
287
|
+
cdp_url: str | None = None,
|
|
288
|
+
is_local: bool = False,
|
|
289
|
+
browser_profile: BrowserProfile | None = None,
|
|
290
|
+
# Cloud browser params (don't mix with local browser params)
|
|
291
|
+
cloud_profile_id: UUID | str | None = None,
|
|
292
|
+
cloud_proxy_country_code: ProxyCountryCode | None = None,
|
|
293
|
+
cloud_timeout: int | None = None,
|
|
294
|
+
# Backward compatibility aliases for cloud params
|
|
295
|
+
profile_id: UUID | str | None = None,
|
|
296
|
+
proxy_country_code: ProxyCountryCode | None = None,
|
|
297
|
+
timeout: int | None = None,
|
|
298
|
+
# BrowserProfile fields that can be passed directly
|
|
299
|
+
# From BrowserConnectArgs
|
|
300
|
+
headers: dict[str, str] | None = None,
|
|
301
|
+
# From BrowserLaunchArgs
|
|
302
|
+
env: dict[str, str | float | bool] | None = None,
|
|
303
|
+
executable_path: str | Path | None = None,
|
|
304
|
+
headless: bool | None = None,
|
|
305
|
+
args: list[str] | None = None,
|
|
306
|
+
ignore_default_args: list[str] | Literal[True] | None = None,
|
|
307
|
+
channel: str | None = None,
|
|
308
|
+
chromium_sandbox: bool | None = None,
|
|
309
|
+
devtools: bool | None = None,
|
|
310
|
+
downloads_path: str | Path | None = None,
|
|
311
|
+
traces_dir: str | Path | None = None,
|
|
312
|
+
# From BrowserContextArgs
|
|
313
|
+
accept_downloads: bool | None = None,
|
|
314
|
+
permissions: list[str] | None = None,
|
|
315
|
+
user_agent: str | None = None,
|
|
316
|
+
screen: dict | None = None,
|
|
317
|
+
viewport: dict | None = None,
|
|
318
|
+
no_viewport: bool | None = None,
|
|
319
|
+
device_scale_factor: float | None = None,
|
|
320
|
+
record_har_content: str | None = None,
|
|
321
|
+
record_har_mode: str | None = None,
|
|
322
|
+
record_har_path: str | Path | None = None,
|
|
323
|
+
record_video_dir: str | Path | None = None,
|
|
324
|
+
record_video_framerate: int | None = None,
|
|
325
|
+
record_video_size: dict | None = None,
|
|
326
|
+
# From BrowserLaunchPersistentContextArgs
|
|
327
|
+
user_data_dir: str | Path | None = None,
|
|
328
|
+
# From BrowserNewContextArgs
|
|
329
|
+
storage_state: str | Path | dict[str, Any] | None = None,
|
|
330
|
+
# BrowserProfile specific fields
|
|
331
|
+
## Cloud Browser Fields
|
|
332
|
+
use_cloud: bool | None = None,
|
|
333
|
+
cloud_browser: bool | None = None, # Backward compatibility alias
|
|
334
|
+
cloud_browser_params: CloudBrowserParams | None = None,
|
|
335
|
+
## Other params
|
|
336
|
+
disable_security: bool | None = None,
|
|
337
|
+
deterministic_rendering: bool | None = None,
|
|
338
|
+
allowed_domains: list[str] | None = None,
|
|
339
|
+
keep_alive: bool | None = None,
|
|
340
|
+
proxy: ProxySettings | None = None,
|
|
341
|
+
enable_default_extensions: bool | None = None,
|
|
342
|
+
window_size: dict | None = None,
|
|
343
|
+
window_position: dict | None = None,
|
|
344
|
+
minimum_wait_page_load_time: float | None = None,
|
|
345
|
+
wait_for_network_idle_page_load_time: float | None = None,
|
|
346
|
+
wait_between_actions: float | None = None,
|
|
347
|
+
filter_highlight_ids: bool | None = None,
|
|
348
|
+
auto_download_pdfs: bool | None = None,
|
|
349
|
+
profile_directory: str | None = None,
|
|
350
|
+
cookie_whitelist_domains: list[str] | None = None,
|
|
351
|
+
# DOM extraction layer configuration
|
|
352
|
+
cross_origin_iframes: bool | None = None,
|
|
353
|
+
highlight_elements: bool | None = None,
|
|
354
|
+
dom_highlight_elements: bool | None = None,
|
|
355
|
+
paint_order_filtering: bool | None = None,
|
|
356
|
+
# Iframe processing limits
|
|
357
|
+
max_iframes: int | None = None,
|
|
358
|
+
max_iframe_depth: int | None = None,
|
|
359
|
+
):
|
|
360
|
+
# Following the same pattern as AgentSettings in service.py
|
|
361
|
+
# Only pass non-None values to avoid validation errors
|
|
362
|
+
profile_kwargs = {
|
|
363
|
+
k: v
|
|
364
|
+
for k, v in locals().items()
|
|
365
|
+
if k
|
|
366
|
+
not in [
|
|
367
|
+
'self',
|
|
368
|
+
'browser_profile',
|
|
369
|
+
'id',
|
|
370
|
+
'cloud_profile_id',
|
|
371
|
+
'cloud_proxy_country_code',
|
|
372
|
+
'cloud_timeout',
|
|
373
|
+
'profile_id',
|
|
374
|
+
'proxy_country_code',
|
|
375
|
+
'timeout',
|
|
376
|
+
]
|
|
377
|
+
and v is not None
|
|
378
|
+
}
|
|
379
|
+
|
|
380
|
+
# Handle backward compatibility: prefer cloud_* params over old names
|
|
381
|
+
final_profile_id = cloud_profile_id if cloud_profile_id is not None else profile_id
|
|
382
|
+
final_proxy_country_code = cloud_proxy_country_code if cloud_proxy_country_code is not None else proxy_country_code
|
|
383
|
+
final_timeout = cloud_timeout if cloud_timeout is not None else timeout
|
|
384
|
+
|
|
385
|
+
# If any cloud params are provided, create cloud_browser_params
|
|
386
|
+
if final_profile_id is not None or final_proxy_country_code is not None or final_timeout is not None:
|
|
387
|
+
cloud_params = CreateBrowserRequest(
|
|
388
|
+
cloud_profile_id=final_profile_id,
|
|
389
|
+
cloud_proxy_country_code=final_proxy_country_code,
|
|
390
|
+
cloud_timeout=final_timeout,
|
|
391
|
+
)
|
|
392
|
+
profile_kwargs['cloud_browser_params'] = cloud_params
|
|
393
|
+
profile_kwargs['use_cloud'] = True
|
|
394
|
+
|
|
395
|
+
# Handle backward compatibility: map cloud_browser to use_cloud
|
|
396
|
+
if 'cloud_browser' in profile_kwargs:
|
|
397
|
+
profile_kwargs['use_cloud'] = profile_kwargs.pop('cloud_browser')
|
|
398
|
+
|
|
399
|
+
# If cloud_browser_params is set, force use_cloud=True
|
|
400
|
+
if cloud_browser_params is not None:
|
|
401
|
+
profile_kwargs['use_cloud'] = True
|
|
402
|
+
|
|
403
|
+
# if is_local is False but executable_path is provided, set is_local to True
|
|
404
|
+
if is_local is False and executable_path is not None:
|
|
405
|
+
profile_kwargs['is_local'] = True
|
|
406
|
+
# Only set is_local=True when cdp_url is missing if we're not using cloud browser
|
|
407
|
+
# (cloud browser will provide cdp_url later)
|
|
408
|
+
use_cloud = profile_kwargs.get('use_cloud') or profile_kwargs.get('cloud_browser')
|
|
409
|
+
if not cdp_url and not use_cloud:
|
|
410
|
+
profile_kwargs['is_local'] = True
|
|
411
|
+
|
|
412
|
+
# Create browser profile from direct parameters or use provided one
|
|
413
|
+
if browser_profile is not None:
|
|
414
|
+
# Merge any direct kwargs into the provided browser_profile (direct kwargs take precedence)
|
|
415
|
+
merged_kwargs = {**browser_profile.model_dump(exclude_unset=True), **profile_kwargs}
|
|
416
|
+
resolved_browser_profile = BrowserProfile(**merged_kwargs)
|
|
417
|
+
else:
|
|
418
|
+
resolved_browser_profile = BrowserProfile(**profile_kwargs)
|
|
419
|
+
|
|
420
|
+
# Initialize the Pydantic model
|
|
421
|
+
super().__init__(
|
|
422
|
+
id=id or str(uuid7str()),
|
|
423
|
+
browser_profile=resolved_browser_profile,
|
|
424
|
+
)
|
|
425
|
+
|
|
426
|
+
# Session configuration (session identity only)
|
|
427
|
+
id: str = Field(default_factory=lambda: str(uuid7str()), description='Unique identifier for this browser session')
|
|
428
|
+
|
|
429
|
+
# Browser configuration (reusable profile)
|
|
430
|
+
browser_profile: BrowserProfile = Field(
|
|
431
|
+
default_factory=lambda: DEFAULT_BROWSER_PROFILE,
|
|
432
|
+
description='BrowserProfile() options to use for the session, otherwise a default profile will be used',
|
|
433
|
+
)
|
|
434
|
+
|
|
435
|
+
# Convenience properties for common browser settings
|
|
436
|
+
@property
|
|
437
|
+
def cdp_url(self) -> str | None:
|
|
438
|
+
"""CDP URL from browser profile."""
|
|
439
|
+
return self.browser_profile.cdp_url
|
|
440
|
+
|
|
441
|
+
@property
|
|
442
|
+
def is_local(self) -> bool:
|
|
443
|
+
"""Whether this is a local browser instance from browser profile."""
|
|
444
|
+
return self.browser_profile.is_local
|
|
445
|
+
|
|
446
|
+
@property
|
|
447
|
+
def cloud_browser(self) -> bool:
|
|
448
|
+
"""Whether to use cloud browser service from browser profile."""
|
|
449
|
+
return self.browser_profile.use_cloud
|
|
450
|
+
|
|
451
|
+
# Main shared event bus for all browser session + all watchdogs
|
|
452
|
+
event_bus: EventBus = Field(default_factory=EventBus)
|
|
453
|
+
|
|
454
|
+
# Mutable public state
|
|
455
|
+
agent_focus: CDPSession | None = None
|
|
456
|
+
|
|
457
|
+
# Mutable private state shared between watchdogs
|
|
458
|
+
_cdp_client_root: CDPClient | None = PrivateAttr(default=None)
|
|
459
|
+
_cdp_session_pool: dict[str, CDPSession] = PrivateAttr(default_factory=dict)
|
|
460
|
+
_session_manager: Any = PrivateAttr(default=None) # SessionManager instance
|
|
461
|
+
_cached_browser_state_summary: Any = PrivateAttr(default=None)
|
|
462
|
+
_cached_selector_map: dict[int, EnhancedDOMTreeNode] = PrivateAttr(default_factory=dict)
|
|
463
|
+
_downloaded_files: list[str] = PrivateAttr(default_factory=list) # Track files downloaded during this session
|
|
464
|
+
_closed_popup_messages: list[str] = PrivateAttr(default_factory=list) # Store messages from auto-closed JavaScript dialogs
|
|
465
|
+
|
|
466
|
+
# Watchdogs
|
|
467
|
+
_crash_watchdog: Any | None = PrivateAttr(default=None)
|
|
468
|
+
_downloads_watchdog: Any | None = PrivateAttr(default=None)
|
|
469
|
+
_aboutblank_watchdog: Any | None = PrivateAttr(default=None)
|
|
470
|
+
_security_watchdog: Any | None = PrivateAttr(default=None)
|
|
471
|
+
_storage_state_watchdog: Any | None = PrivateAttr(default=None)
|
|
472
|
+
_local_browser_watchdog: Any | None = PrivateAttr(default=None)
|
|
473
|
+
_default_action_watchdog: Any | None = PrivateAttr(default=None)
|
|
474
|
+
_dom_watchdog: Any | None = PrivateAttr(default=None)
|
|
475
|
+
_screenshot_watchdog: Any | None = PrivateAttr(default=None)
|
|
476
|
+
_permissions_watchdog: Any | None = PrivateAttr(default=None)
|
|
477
|
+
_recording_watchdog: Any | None = PrivateAttr(default=None)
|
|
478
|
+
|
|
479
|
+
_cloud_browser_client: CloudBrowserClient = PrivateAttr(default_factory=lambda: CloudBrowserClient())
|
|
480
|
+
|
|
481
|
+
_logger: Any = PrivateAttr(default=None)
|
|
482
|
+
|
|
483
|
+
@property
|
|
484
|
+
def logger(self) -> Any:
|
|
485
|
+
"""Get instance-specific logger with session ID in the name"""
|
|
486
|
+
# **regenerate it every time** because our id and str(self) can change as browser connection state changes
|
|
487
|
+
# if self._logger is None or not self._cdp_client_root:
|
|
488
|
+
# self._logger = logging.getLogger(f'browser_use.{self}')
|
|
489
|
+
return logging.getLogger(f'browser_use.{self}')
|
|
490
|
+
|
|
491
|
+
@cached_property
|
|
492
|
+
def _id_for_logs(self) -> str:
|
|
493
|
+
"""Get human-friendly semi-unique identifier for differentiating different BrowserSession instances in logs"""
|
|
494
|
+
str_id = self.id[-4:] # default to last 4 chars of truly random uuid, less helpful than cdp port but always unique enough
|
|
495
|
+
port_number = (self.cdp_url or 'no-cdp').rsplit(':', 1)[-1].split('/', 1)[0].strip()
|
|
496
|
+
port_is_random = not port_number.startswith('922')
|
|
497
|
+
port_is_unique_enough = port_number not in _LOGGED_UNIQUE_SESSION_IDS
|
|
498
|
+
if port_number and port_number.isdigit() and port_is_random and port_is_unique_enough:
|
|
499
|
+
# if cdp port is random/unique enough to identify this session, use it as our id in logs
|
|
500
|
+
_LOGGED_UNIQUE_SESSION_IDS.add(port_number)
|
|
501
|
+
str_id = port_number
|
|
502
|
+
return str_id
|
|
503
|
+
|
|
504
|
+
@property
|
|
505
|
+
def _tab_id_for_logs(self) -> str:
|
|
506
|
+
return self.agent_focus.target_id[-2:] if self.agent_focus and self.agent_focus.target_id else f'{red}--{reset}'
|
|
507
|
+
|
|
508
|
+
def __repr__(self) -> str:
|
|
509
|
+
return f'BrowserSession🅑 {self._id_for_logs} 🅣 {self._tab_id_for_logs} (cdp_url={self.cdp_url}, profile={self.browser_profile})'
|
|
510
|
+
|
|
511
|
+
def __str__(self) -> str:
|
|
512
|
+
return f'BrowserSession🅑 {self._id_for_logs} 🅣 {self._tab_id_for_logs}'
|
|
513
|
+
|
|
514
|
+
async def reset(self) -> None:
|
|
515
|
+
"""Clear all cached CDP sessions with proper cleanup."""
|
|
516
|
+
|
|
517
|
+
# TODO: clear the event bus queue here, implement this helper
|
|
518
|
+
# await self.event_bus.wait_for_idle(timeout=5.0)
|
|
519
|
+
# await self.event_bus.clear()
|
|
520
|
+
|
|
521
|
+
# Clear session manager first (stops event monitoring)
|
|
522
|
+
if self._session_manager:
|
|
523
|
+
await self._session_manager.clear()
|
|
524
|
+
self._session_manager = None
|
|
525
|
+
|
|
526
|
+
# Clear session pool (all sessions share the root WebSocket, so no disconnect needed)
|
|
527
|
+
self._cdp_session_pool.clear()
|
|
528
|
+
|
|
529
|
+
self._cdp_client_root = None # type: ignore
|
|
530
|
+
self._cached_browser_state_summary = None
|
|
531
|
+
self._cached_selector_map.clear()
|
|
532
|
+
self._downloaded_files.clear()
|
|
533
|
+
|
|
534
|
+
self.agent_focus = None
|
|
535
|
+
if self.is_local:
|
|
536
|
+
self.browser_profile.cdp_url = None
|
|
537
|
+
|
|
538
|
+
self._crash_watchdog = None
|
|
539
|
+
self._downloads_watchdog = None
|
|
540
|
+
self._aboutblank_watchdog = None
|
|
541
|
+
self._security_watchdog = None
|
|
542
|
+
self._storage_state_watchdog = None
|
|
543
|
+
self._local_browser_watchdog = None
|
|
544
|
+
self._default_action_watchdog = None
|
|
545
|
+
self._dom_watchdog = None
|
|
546
|
+
self._screenshot_watchdog = None
|
|
547
|
+
self._permissions_watchdog = None
|
|
548
|
+
self._recording_watchdog = None
|
|
549
|
+
|
|
550
|
+
def model_post_init(self, __context) -> None:
|
|
551
|
+
"""Register event handlers after model initialization."""
|
|
552
|
+
# Check if handlers are already registered to prevent duplicates
|
|
553
|
+
|
|
554
|
+
from browser_use.browser.watchdog_base import BaseWatchdog
|
|
555
|
+
|
|
556
|
+
start_handlers = self.event_bus.handlers.get('BrowserStartEvent', [])
|
|
557
|
+
start_handler_names = [getattr(h, '__name__', str(h)) for h in start_handlers]
|
|
558
|
+
|
|
559
|
+
if any('on_BrowserStartEvent' in name for name in start_handler_names):
|
|
560
|
+
raise RuntimeError(
|
|
561
|
+
'[BrowserSession] Duplicate handler registration attempted! '
|
|
562
|
+
'on_BrowserStartEvent is already registered. '
|
|
563
|
+
'This likely means BrowserSession was initialized multiple times with the same EventBus.'
|
|
564
|
+
)
|
|
565
|
+
|
|
566
|
+
BaseWatchdog.attach_handler_to_session(self, BrowserStartEvent, self.on_BrowserStartEvent)
|
|
567
|
+
BaseWatchdog.attach_handler_to_session(self, BrowserStopEvent, self.on_BrowserStopEvent)
|
|
568
|
+
# BaseWatchdog.attach_handler_to_session(self, NavigateToUrlEvent, self.on_NavigateToUrlEvent)
|
|
569
|
+
BaseWatchdog.attach_handler_to_session(self, SwitchTabEvent, self.on_SwitchTabEvent)
|
|
570
|
+
# BaseWatchdog.attach_handler_to_session(self, TabCreatedEvent, self.on_TabCreatedEvent)
|
|
571
|
+
# BaseWatchdog.attach_handler_to_session(self, TabClosedEvent, self.on_TabClosedEvent)
|
|
572
|
+
BaseWatchdog.attach_handler_to_session(self, AgentFocusChangedEvent, self.on_AgentFocusChangedEvent)
|
|
573
|
+
# BaseWatchdog.attach_handler_to_session(self, FileDownloadedEvent, self.on_FileDownloadedEvent)
|
|
574
|
+
# BaseWatchdog.attach_handler_to_session(self, CloseTabEvent, self.on_CloseTabEvent)
|
|
575
|
+
|
|
576
|
+
@observe_debug(ignore_input=True, ignore_output=True, name='browser_session_start')
|
|
577
|
+
async def start(self) -> None:
|
|
578
|
+
"""Start the browser session."""
|
|
579
|
+
start_event = self.event_bus.dispatch(BrowserStartEvent())
|
|
580
|
+
await start_event
|
|
581
|
+
# Ensure any exceptions from the event handler are propagated
|
|
582
|
+
await start_event.event_result(raise_if_any=True, raise_if_none=False)
|
|
583
|
+
|
|
584
|
+
async def kill(self) -> None:
|
|
585
|
+
"""Kill the browser session and reset all state."""
|
|
586
|
+
# First save storage state while CDP is still connected
|
|
587
|
+
from browser_use.browser.events import SaveStorageStateEvent
|
|
588
|
+
|
|
589
|
+
save_event = self.event_bus.dispatch(SaveStorageStateEvent())
|
|
590
|
+
await save_event
|
|
591
|
+
|
|
592
|
+
# Dispatch stop event to kill the browser
|
|
593
|
+
await self.event_bus.dispatch(BrowserStopEvent(force=True))
|
|
594
|
+
# Stop the event bus
|
|
595
|
+
await self.event_bus.stop(clear=True, timeout=5)
|
|
596
|
+
# Reset all state
|
|
597
|
+
await self.reset()
|
|
598
|
+
# Create fresh event bus
|
|
599
|
+
self.event_bus = EventBus()
|
|
600
|
+
|
|
601
|
+
async def stop(self) -> None:
|
|
602
|
+
"""Stop the browser session without killing the browser process.
|
|
603
|
+
|
|
604
|
+
This clears event buses and cached state but keeps the browser alive.
|
|
605
|
+
Useful when you want to clean up resources but plan to reconnect later.
|
|
606
|
+
"""
|
|
607
|
+
# First save storage state while CDP is still connected
|
|
608
|
+
from browser_use.browser.events import SaveStorageStateEvent
|
|
609
|
+
|
|
610
|
+
save_event = self.event_bus.dispatch(SaveStorageStateEvent())
|
|
611
|
+
await save_event
|
|
612
|
+
|
|
613
|
+
# Now dispatch BrowserStopEvent to notify watchdogs
|
|
614
|
+
await self.event_bus.dispatch(BrowserStopEvent(force=False))
|
|
615
|
+
|
|
616
|
+
# Stop the event bus
|
|
617
|
+
await self.event_bus.stop(clear=True, timeout=5)
|
|
618
|
+
# Reset all state
|
|
619
|
+
await self.reset()
|
|
620
|
+
# Create fresh event bus
|
|
621
|
+
self.event_bus = EventBus()
|
|
622
|
+
|
|
623
|
+
@observe_debug(ignore_input=True, ignore_output=True, name='browser_start_event_handler')
|
|
624
|
+
async def on_BrowserStartEvent(self, event: BrowserStartEvent) -> dict[str, str]:
|
|
625
|
+
"""Handle browser start request.
|
|
626
|
+
|
|
627
|
+
Returns:
|
|
628
|
+
Dict with 'cdp_url' key containing the CDP URL
|
|
629
|
+
"""
|
|
630
|
+
|
|
631
|
+
# await self.reset()
|
|
632
|
+
|
|
633
|
+
# Initialize and attach all watchdogs FIRST so LocalBrowserWatchdog can handle BrowserLaunchEvent
|
|
634
|
+
await self.attach_all_watchdogs()
|
|
635
|
+
|
|
636
|
+
try:
|
|
637
|
+
# If no CDP URL, launch local browser or cloud browser
|
|
638
|
+
if not self.cdp_url:
|
|
639
|
+
if self.browser_profile.use_cloud or self.browser_profile.cloud_browser_params is not None:
|
|
640
|
+
# Use cloud browser service
|
|
641
|
+
try:
|
|
642
|
+
# Use cloud_browser_params if provided, otherwise create empty request
|
|
643
|
+
cloud_params = self.browser_profile.cloud_browser_params or CreateBrowserRequest()
|
|
644
|
+
cloud_browser_response = await self._cloud_browser_client.create_browser(cloud_params)
|
|
645
|
+
self.browser_profile.cdp_url = cloud_browser_response.cdpUrl
|
|
646
|
+
self.browser_profile.is_local = False
|
|
647
|
+
self.logger.info('🌤️ Successfully connected to cloud browser service')
|
|
648
|
+
except CloudBrowserAuthError:
|
|
649
|
+
raise CloudBrowserAuthError(
|
|
650
|
+
'Authentication failed for cloud browser service. Set BROWSER_USE_API_KEY environment variable. You can also create an API key at https://cloud.browser-use.com/new-api-key'
|
|
651
|
+
)
|
|
652
|
+
except CloudBrowserError as e:
|
|
653
|
+
raise CloudBrowserError(f'Failed to create cloud browser: {e}')
|
|
654
|
+
elif self.is_local:
|
|
655
|
+
# Launch local browser using event-driven approach
|
|
656
|
+
launch_event = self.event_bus.dispatch(BrowserLaunchEvent())
|
|
657
|
+
await launch_event
|
|
658
|
+
|
|
659
|
+
# Get the CDP URL from LocalBrowserWatchdog handler result
|
|
660
|
+
launch_result: BrowserLaunchResult = cast(
|
|
661
|
+
BrowserLaunchResult, await launch_event.event_result(raise_if_none=True, raise_if_any=True)
|
|
662
|
+
)
|
|
663
|
+
self.browser_profile.cdp_url = launch_result.cdp_url
|
|
664
|
+
else:
|
|
665
|
+
raise ValueError('Got BrowserSession(is_local=False) but no cdp_url was provided to connect to!')
|
|
666
|
+
|
|
667
|
+
assert self.cdp_url and '://' in self.cdp_url
|
|
668
|
+
|
|
669
|
+
# Only connect if not already connected
|
|
670
|
+
if self._cdp_client_root is None:
|
|
671
|
+
# Setup browser via CDP (for both local and remote cases)
|
|
672
|
+
await self.connect(cdp_url=self.cdp_url)
|
|
673
|
+
assert self.cdp_client is not None
|
|
674
|
+
|
|
675
|
+
# Notify that browser is connected (single place)
|
|
676
|
+
self.event_bus.dispatch(BrowserConnectedEvent(cdp_url=self.cdp_url))
|
|
677
|
+
else:
|
|
678
|
+
self.logger.debug('Already connected to CDP, skipping reconnection')
|
|
679
|
+
|
|
680
|
+
# Return the CDP URL for other components
|
|
681
|
+
return {'cdp_url': self.cdp_url}
|
|
682
|
+
|
|
683
|
+
except Exception as e:
|
|
684
|
+
self.event_bus.dispatch(
|
|
685
|
+
BrowserErrorEvent(
|
|
686
|
+
error_type='BrowserStartEventError',
|
|
687
|
+
message=f'Failed to start browser: {type(e).__name__} {e}',
|
|
688
|
+
details={'cdp_url': self.cdp_url, 'is_local': self.is_local},
|
|
689
|
+
)
|
|
690
|
+
)
|
|
691
|
+
raise
|
|
692
|
+
|
|
693
|
+
async def on_NavigateToUrlEvent(self, event: NavigateToUrlEvent) -> None:
|
|
694
|
+
"""Handle navigation requests - core browser functionality."""
|
|
695
|
+
self.logger.debug(f'[on_NavigateToUrlEvent] Received NavigateToUrlEvent: url={event.url}, new_tab={event.new_tab}')
|
|
696
|
+
if not self.agent_focus:
|
|
697
|
+
self.logger.warning('Cannot navigate - browser not connected')
|
|
698
|
+
return
|
|
699
|
+
|
|
700
|
+
target_id = None
|
|
701
|
+
|
|
702
|
+
# If new_tab=True but we're already in a new tab, set new_tab=False
|
|
703
|
+
if event.new_tab:
|
|
704
|
+
try:
|
|
705
|
+
current_url = await self.get_current_page_url()
|
|
706
|
+
from browser_use.utils import is_new_tab_page
|
|
707
|
+
|
|
708
|
+
if is_new_tab_page(current_url):
|
|
709
|
+
self.logger.debug(f'[on_NavigateToUrlEvent] Already in new tab ({current_url}), setting new_tab=False')
|
|
710
|
+
event.new_tab = False
|
|
711
|
+
except Exception as e:
|
|
712
|
+
self.logger.debug(f'[on_NavigateToUrlEvent] Could not check current URL: {e}')
|
|
713
|
+
|
|
714
|
+
# check if the url is already open in a tab somewhere that we're not currently on, if so, short-circuit and just switch to it
|
|
715
|
+
targets = await self._cdp_get_all_pages()
|
|
716
|
+
for target in targets:
|
|
717
|
+
if target.get('url') == event.url and target['targetId'] != self.agent_focus.target_id and not event.new_tab:
|
|
718
|
+
target_id = target['targetId']
|
|
719
|
+
event.new_tab = False
|
|
720
|
+
# await self.event_bus.dispatch(SwitchTabEvent(target_id=target_id))
|
|
721
|
+
|
|
722
|
+
try:
|
|
723
|
+
# Find or create target for navigation
|
|
724
|
+
|
|
725
|
+
self.logger.debug(f'[on_NavigateToUrlEvent] Processing new_tab={event.new_tab}')
|
|
726
|
+
if event.new_tab:
|
|
727
|
+
# Look for existing about:blank tab that's not the current one
|
|
728
|
+
targets = await self._cdp_get_all_pages()
|
|
729
|
+
self.logger.debug(f'[on_NavigateToUrlEvent] Found {len(targets)} existing tabs')
|
|
730
|
+
current_target_id = self.agent_focus.target_id if self.agent_focus else None
|
|
731
|
+
self.logger.debug(f'[on_NavigateToUrlEvent] Current target_id: {current_target_id}')
|
|
732
|
+
|
|
733
|
+
for idx, target in enumerate(targets):
|
|
734
|
+
self.logger.debug(
|
|
735
|
+
f'[on_NavigateToUrlEvent] Tab {idx}: url={target.get("url")}, targetId={target["targetId"]}'
|
|
736
|
+
)
|
|
737
|
+
if target.get('url') == 'about:blank' and target['targetId'] != current_target_id:
|
|
738
|
+
target_id = target['targetId']
|
|
739
|
+
self.logger.debug(f'Reusing existing about:blank tab #{target_id[-4:]}')
|
|
740
|
+
break
|
|
741
|
+
|
|
742
|
+
# Create new tab if no reusable one found
|
|
743
|
+
if not target_id:
|
|
744
|
+
self.logger.debug('[on_NavigateToUrlEvent] No reusable about:blank tab found, creating new tab...')
|
|
745
|
+
try:
|
|
746
|
+
target_id = await self._cdp_create_new_page('about:blank')
|
|
747
|
+
self.logger.debug(f'[on_NavigateToUrlEvent] Created new page with target_id: {target_id}')
|
|
748
|
+
targets = await self._cdp_get_all_pages()
|
|
749
|
+
|
|
750
|
+
self.logger.debug(f'Created new tab #{target_id[-4:]}')
|
|
751
|
+
# Dispatch TabCreatedEvent for new tab
|
|
752
|
+
await self.event_bus.dispatch(TabCreatedEvent(target_id=target_id, url='about:blank'))
|
|
753
|
+
except Exception as e:
|
|
754
|
+
self.logger.error(f'[on_NavigateToUrlEvent] Failed to create new tab: {type(e).__name__}: {e}')
|
|
755
|
+
# Fall back to using current tab
|
|
756
|
+
target_id = self.agent_focus.target_id
|
|
757
|
+
self.logger.warning(f'[on_NavigateToUrlEvent] Falling back to current tab #{target_id[-4:]}')
|
|
758
|
+
else:
|
|
759
|
+
# Use current tab
|
|
760
|
+
target_id = target_id or self.agent_focus.target_id
|
|
761
|
+
|
|
762
|
+
# Only switch tab if we're not already on the target tab
|
|
763
|
+
if self.agent_focus is None or self.agent_focus.target_id != target_id:
|
|
764
|
+
self.logger.debug(
|
|
765
|
+
f'[on_NavigateToUrlEvent] Switching to target tab {target_id[-4:]} (current: {self.agent_focus.target_id[-4:] if self.agent_focus else "none"})'
|
|
766
|
+
)
|
|
767
|
+
# Activate target (bring to foreground)
|
|
768
|
+
await self.event_bus.dispatch(SwitchTabEvent(target_id=target_id))
|
|
769
|
+
# which does this for us:
|
|
770
|
+
# self.agent_focus = await self.get_or_create_cdp_session(target_id)
|
|
771
|
+
else:
|
|
772
|
+
self.logger.debug(f'[on_NavigateToUrlEvent] Already on target tab {target_id[-4:]}, skipping SwitchTabEvent')
|
|
773
|
+
|
|
774
|
+
assert self.agent_focus is not None and self.agent_focus.target_id == target_id, (
|
|
775
|
+
'Agent focus not updated to new target_id after SwitchTabEvent should have switched to it'
|
|
776
|
+
)
|
|
777
|
+
|
|
778
|
+
# Dispatch navigation started
|
|
779
|
+
await self.event_bus.dispatch(NavigationStartedEvent(target_id=target_id, url=event.url))
|
|
780
|
+
|
|
781
|
+
# Navigate to URL
|
|
782
|
+
await self.agent_focus.cdp_client.send.Page.navigate(
|
|
783
|
+
params={
|
|
784
|
+
'url': event.url,
|
|
785
|
+
'transitionType': 'address_bar',
|
|
786
|
+
# 'referrer': 'https://www.google.com',
|
|
787
|
+
},
|
|
788
|
+
session_id=self.agent_focus.session_id,
|
|
789
|
+
)
|
|
790
|
+
|
|
791
|
+
# # Wait a bit to ensure page starts loading
|
|
792
|
+
await asyncio.sleep(1)
|
|
793
|
+
|
|
794
|
+
# Close any extension options pages that might have opened
|
|
795
|
+
await self._close_extension_options_pages()
|
|
796
|
+
|
|
797
|
+
# Dispatch navigation complete
|
|
798
|
+
self.logger.debug(f'Dispatching NavigationCompleteEvent for {event.url} (tab #{target_id[-4:]})')
|
|
799
|
+
await self.event_bus.dispatch(
|
|
800
|
+
NavigationCompleteEvent(
|
|
801
|
+
target_id=target_id,
|
|
802
|
+
url=event.url,
|
|
803
|
+
status=None, # CDP doesn't provide status directly
|
|
804
|
+
)
|
|
805
|
+
)
|
|
806
|
+
await self.event_bus.dispatch(
|
|
807
|
+
AgentFocusChangedEvent(target_id=target_id, url=event.url)
|
|
808
|
+
) # do not await! AgentFocusChangedEvent calls SwitchTabEvent and it will deadlock, dispatch to enqueue and return
|
|
809
|
+
|
|
810
|
+
# Note: These should be handled by dedicated watchdogs:
|
|
811
|
+
# - Security checks (security_watchdog)
|
|
812
|
+
# - Page health checks (crash_watchdog)
|
|
813
|
+
# - Dialog handling (dialog_watchdog)
|
|
814
|
+
# - Download handling (downloads_watchdog)
|
|
815
|
+
# - DOM rebuilding (dom_watchdog)
|
|
816
|
+
|
|
817
|
+
except Exception as e:
|
|
818
|
+
self.logger.error(f'Navigation failed: {type(e).__name__}: {e}')
|
|
819
|
+
if target_id:
|
|
820
|
+
await self.event_bus.dispatch(
|
|
821
|
+
NavigationCompleteEvent(
|
|
822
|
+
target_id=target_id,
|
|
823
|
+
url=event.url,
|
|
824
|
+
error_message=f'{type(e).__name__}: {e}',
|
|
825
|
+
)
|
|
826
|
+
)
|
|
827
|
+
await self.event_bus.dispatch(AgentFocusChangedEvent(target_id=target_id, url=event.url))
|
|
828
|
+
raise
|
|
829
|
+
|
|
830
|
+
async def on_SwitchTabEvent(self, event: SwitchTabEvent) -> TargetID:
|
|
831
|
+
"""Handle tab switching - core browser functionality."""
|
|
832
|
+
if not self.agent_focus:
|
|
833
|
+
raise RuntimeError('Cannot switch tabs - browser not connected')
|
|
834
|
+
|
|
835
|
+
all_pages = await self._cdp_get_all_pages()
|
|
836
|
+
if event.target_id is None:
|
|
837
|
+
# most recently opened page
|
|
838
|
+
if all_pages:
|
|
839
|
+
# update the target id to be the id of the most recently opened page, then proceed to switch to it
|
|
840
|
+
event.target_id = all_pages[-1]['targetId']
|
|
841
|
+
else:
|
|
842
|
+
# no pages open at all, create a new one (handles switching to it automatically)
|
|
843
|
+
assert self._cdp_client_root is not None, 'CDP client root not initialized - browser may not be connected yet'
|
|
844
|
+
new_target = await self._cdp_client_root.send.Target.createTarget(params={'url': 'about:blank'})
|
|
845
|
+
target_id = new_target['targetId']
|
|
846
|
+
# do not await! these may circularly trigger SwitchTabEvent and could deadlock, dispatch to enqueue and return
|
|
847
|
+
self.event_bus.dispatch(TabCreatedEvent(url='about:blank', target_id=target_id))
|
|
848
|
+
self.event_bus.dispatch(AgentFocusChangedEvent(target_id=target_id, url='about:blank'))
|
|
849
|
+
return target_id
|
|
850
|
+
|
|
851
|
+
# switch to the target
|
|
852
|
+
self.agent_focus = await self.get_or_create_cdp_session(target_id=event.target_id, focus=True)
|
|
853
|
+
|
|
854
|
+
# Visually switch to the tab in the browser
|
|
855
|
+
# The Force Background Tab extension prevents Chrome from auto-switching when links create new tabs,
|
|
856
|
+
# but we still want the agent to be able to explicitly switch tabs when needed
|
|
857
|
+
await self.agent_focus.cdp_client.send.Target.activateTarget(params={'targetId': event.target_id})
|
|
858
|
+
|
|
859
|
+
# dispatch focus changed event
|
|
860
|
+
await self.event_bus.dispatch(
|
|
861
|
+
AgentFocusChangedEvent(
|
|
862
|
+
target_id=self.agent_focus.target_id,
|
|
863
|
+
url=self.agent_focus.url,
|
|
864
|
+
)
|
|
865
|
+
)
|
|
866
|
+
return self.agent_focus.target_id
|
|
867
|
+
|
|
868
|
+
async def on_CloseTabEvent(self, event: CloseTabEvent) -> None:
|
|
869
|
+
"""Handle tab closure - update focus if needed."""
|
|
870
|
+
try:
|
|
871
|
+
# Dispatch tab closed event
|
|
872
|
+
await self.event_bus.dispatch(TabClosedEvent(target_id=event.target_id))
|
|
873
|
+
|
|
874
|
+
# Try to close the target, but don't fail if it's already closed
|
|
875
|
+
try:
|
|
876
|
+
cdp_session = await self.get_or_create_cdp_session(target_id=None, focus=False)
|
|
877
|
+
await cdp_session.cdp_client.send.Target.closeTarget(params={'targetId': event.target_id})
|
|
878
|
+
except Exception as e:
|
|
879
|
+
self.logger.debug(f'Target may already be closed: {e}')
|
|
880
|
+
except Exception as e:
|
|
881
|
+
self.logger.warning(f'Error during tab close cleanup: {e}')
|
|
882
|
+
|
|
883
|
+
async def on_TabCreatedEvent(self, event: TabCreatedEvent) -> None:
|
|
884
|
+
"""Handle tab creation - apply viewport settings to new tab."""
|
|
885
|
+
# Note: Tab switching prevention is handled by the Force Background Tab extension
|
|
886
|
+
# The extension automatically keeps focus on the current tab when new tabs are created
|
|
887
|
+
|
|
888
|
+
# Apply viewport settings if configured
|
|
889
|
+
if self.browser_profile.viewport and not self.browser_profile.no_viewport:
|
|
890
|
+
try:
|
|
891
|
+
viewport_width = self.browser_profile.viewport.width
|
|
892
|
+
viewport_height = self.browser_profile.viewport.height
|
|
893
|
+
device_scale_factor = self.browser_profile.device_scale_factor or 1.0
|
|
894
|
+
|
|
895
|
+
# Use the helper method with the new tab's target_id
|
|
896
|
+
await self._cdp_set_viewport(viewport_width, viewport_height, device_scale_factor, target_id=event.target_id)
|
|
897
|
+
|
|
898
|
+
self.logger.debug(f'Applied viewport {viewport_width}x{viewport_height} to tab {event.target_id[-8:]}')
|
|
899
|
+
except Exception as e:
|
|
900
|
+
self.logger.warning(f'Failed to set viewport for new tab {event.target_id[-8:]}: {e}')
|
|
901
|
+
|
|
902
|
+
async def on_TabClosedEvent(self, event: TabClosedEvent) -> None:
|
|
903
|
+
"""Handle tab closure - update focus if needed."""
|
|
904
|
+
if not self.agent_focus:
|
|
905
|
+
return
|
|
906
|
+
|
|
907
|
+
# Get current tab index
|
|
908
|
+
current_target_id = self.agent_focus.target_id
|
|
909
|
+
|
|
910
|
+
# If the closed tab was the current one, find a new target
|
|
911
|
+
if current_target_id == event.target_id:
|
|
912
|
+
await self.event_bus.dispatch(SwitchTabEvent(target_id=None))
|
|
913
|
+
|
|
914
|
+
async def on_AgentFocusChangedEvent(self, event: AgentFocusChangedEvent) -> None:
|
|
915
|
+
"""Handle agent focus change - update focus and clear cache."""
|
|
916
|
+
self.logger.debug(f'🔄 AgentFocusChangedEvent received: target_id=...{event.target_id[-4:]} url={event.url}')
|
|
917
|
+
|
|
918
|
+
# Clear cached DOM state since focus changed
|
|
919
|
+
# self.logger.debug('🔄 Clearing DOM cache...')
|
|
920
|
+
if self._dom_watchdog:
|
|
921
|
+
self._dom_watchdog.clear_cache()
|
|
922
|
+
# self.logger.debug('🔄 Cleared DOM cache after focus change')
|
|
923
|
+
|
|
924
|
+
# Clear cached browser state
|
|
925
|
+
# self.logger.debug('🔄 Clearing cached browser state...')
|
|
926
|
+
self._cached_browser_state_summary = None
|
|
927
|
+
self._cached_selector_map.clear()
|
|
928
|
+
self.logger.debug('🔄 Cached browser state cleared')
|
|
929
|
+
all_targets = await self._cdp_get_all_pages(include_chrome=True)
|
|
930
|
+
|
|
931
|
+
# Update agent focus if a specific target_id is provided
|
|
932
|
+
if event.target_id:
|
|
933
|
+
self.agent_focus = await self.get_or_create_cdp_session(target_id=event.target_id, focus=True)
|
|
934
|
+
self.logger.debug(f'🔄 Updated agent focus to tab target_id=...{event.target_id[-4:]}')
|
|
935
|
+
else:
|
|
936
|
+
raise RuntimeError('AgentFocusChangedEvent received with no target_id for newly focused tab')
|
|
937
|
+
|
|
938
|
+
# Test that the browser is responsive by evaluating a simple expression
|
|
939
|
+
if self.agent_focus:
|
|
940
|
+
self.logger.debug('🔄 Testing tab responsiveness...')
|
|
941
|
+
try:
|
|
942
|
+
test_result = await asyncio.wait_for(
|
|
943
|
+
self.agent_focus.cdp_client.send.Runtime.evaluate(
|
|
944
|
+
params={'expression': '1 + 1', 'returnByValue': True}, session_id=self.agent_focus.session_id
|
|
945
|
+
),
|
|
946
|
+
timeout=2.0,
|
|
947
|
+
)
|
|
948
|
+
if test_result.get('result', {}).get('value') == 2:
|
|
949
|
+
# self.logger.debug('🔄 ✅ Browser is responsive after focus change')
|
|
950
|
+
pass
|
|
951
|
+
else:
|
|
952
|
+
raise Exception('❌ Failed to execute test JS expression with Page.evaluate')
|
|
953
|
+
except Exception as e:
|
|
954
|
+
self.logger.error(
|
|
955
|
+
f'🔄 ❌ Target {self.agent_focus.target_id} seems closed/crashed, switching to fallback page {all_targets[0]}: {type(e).__name__}: {e}'
|
|
956
|
+
)
|
|
957
|
+
all_pages = await self._cdp_get_all_pages()
|
|
958
|
+
last_target_id = all_pages[-1]['targetId'] if all_pages else None
|
|
959
|
+
self.agent_focus = await self.get_or_create_cdp_session(target_id=last_target_id, focus=True)
|
|
960
|
+
raise
|
|
961
|
+
|
|
962
|
+
# Dispatch NavigationCompleteEvent when tab focus changes
|
|
963
|
+
# This ensures PDF detection and downloads work when switching tabs
|
|
964
|
+
if event.target_id and event.url:
|
|
965
|
+
self.logger.debug(f'🔄 Dispatching NavigationCompleteEvent for tab switch to {event.url[:50]}...')
|
|
966
|
+
await self.event_bus.dispatch(
|
|
967
|
+
NavigationCompleteEvent(
|
|
968
|
+
target_id=event.target_id,
|
|
969
|
+
url=event.url,
|
|
970
|
+
)
|
|
971
|
+
)
|
|
972
|
+
|
|
973
|
+
# self.logger.debug('🔄 AgentFocusChangedEvent handler completed successfully')
|
|
974
|
+
|
|
975
|
+
async def on_FileDownloadedEvent(self, event: FileDownloadedEvent) -> None:
|
|
976
|
+
"""Track downloaded files during this session."""
|
|
977
|
+
self.logger.debug(f'FileDownloadedEvent received: {event.file_name} at {event.path}')
|
|
978
|
+
if event.path and event.path not in self._downloaded_files:
|
|
979
|
+
self._downloaded_files.append(event.path)
|
|
980
|
+
self.logger.info(f'📁 Tracked download: {event.file_name} ({len(self._downloaded_files)} total downloads in session)')
|
|
981
|
+
else:
|
|
982
|
+
if not event.path:
|
|
983
|
+
self.logger.warning(f'FileDownloadedEvent has no path: {event}')
|
|
984
|
+
else:
|
|
985
|
+
self.logger.debug(f'File already tracked: {event.path}')
|
|
986
|
+
|
|
987
|
+
async def on_BrowserStopEvent(self, event: BrowserStopEvent) -> None:
|
|
988
|
+
"""Handle browser stop request."""
|
|
989
|
+
|
|
990
|
+
try:
|
|
991
|
+
# Check if we should keep the browser alive
|
|
992
|
+
if self.browser_profile.keep_alive and not event.force:
|
|
993
|
+
self.event_bus.dispatch(BrowserStoppedEvent(reason='Kept alive due to keep_alive=True'))
|
|
994
|
+
return
|
|
995
|
+
|
|
996
|
+
# Clean up cloud browser session if using cloud browser
|
|
997
|
+
if self.browser_profile.use_cloud:
|
|
998
|
+
try:
|
|
999
|
+
await self._cloud_browser_client.stop_browser()
|
|
1000
|
+
self.logger.info('🌤️ Cloud browser session cleaned up')
|
|
1001
|
+
except Exception as e:
|
|
1002
|
+
self.logger.debug(f'Failed to cleanup cloud browser session: {e}')
|
|
1003
|
+
|
|
1004
|
+
# Clear CDP session cache before stopping
|
|
1005
|
+
await self.reset()
|
|
1006
|
+
|
|
1007
|
+
# Reset state
|
|
1008
|
+
if self.is_local:
|
|
1009
|
+
self.browser_profile.cdp_url = None
|
|
1010
|
+
|
|
1011
|
+
# Notify stop and wait for all handlers to complete
|
|
1012
|
+
# LocalBrowserWatchdog listens for BrowserStopEvent and dispatches BrowserKillEvent
|
|
1013
|
+
stop_event = self.event_bus.dispatch(BrowserStoppedEvent(reason='Stopped by request'))
|
|
1014
|
+
await stop_event
|
|
1015
|
+
|
|
1016
|
+
except Exception as e:
|
|
1017
|
+
self.event_bus.dispatch(
|
|
1018
|
+
BrowserErrorEvent(
|
|
1019
|
+
error_type='BrowserStopEventError',
|
|
1020
|
+
message=f'Failed to stop browser: {type(e).__name__} {e}',
|
|
1021
|
+
details={'cdp_url': self.cdp_url, 'is_local': self.is_local},
|
|
1022
|
+
)
|
|
1023
|
+
)
|
|
1024
|
+
|
|
1025
|
+
# region - ========== CDP-based replacements for browser_context operations ==========
|
|
1026
|
+
@property
|
|
1027
|
+
def cdp_client(self) -> CDPClient:
|
|
1028
|
+
"""Get the cached root CDP cdp_session.cdp_client. The client is created and started in self.connect()."""
|
|
1029
|
+
assert self._cdp_client_root is not None, 'CDP client not initialized - browser may not be connected yet'
|
|
1030
|
+
return self._cdp_client_root
|
|
1031
|
+
|
|
1032
|
+
async def new_page(self, url: str | None = None) -> 'Page':
|
|
1033
|
+
"""Create a new page (tab)."""
|
|
1034
|
+
from cdp_use.cdp.target.commands import CreateTargetParameters
|
|
1035
|
+
|
|
1036
|
+
params: CreateTargetParameters = {'url': url or 'about:blank'}
|
|
1037
|
+
result = await self.cdp_client.send.Target.createTarget(params)
|
|
1038
|
+
|
|
1039
|
+
target_id = result['targetId']
|
|
1040
|
+
|
|
1041
|
+
# Import here to avoid circular import
|
|
1042
|
+
from browser_use.actor.page import Page as Target
|
|
1043
|
+
|
|
1044
|
+
return Target(self, target_id)
|
|
1045
|
+
|
|
1046
|
+
async def get_current_page(self) -> 'Page | None':
|
|
1047
|
+
"""Get the current page as an actor Page."""
|
|
1048
|
+
target_info = await self.get_current_target_info()
|
|
1049
|
+
|
|
1050
|
+
if not target_info:
|
|
1051
|
+
return None
|
|
1052
|
+
|
|
1053
|
+
from browser_use.actor.page import Page as Target
|
|
1054
|
+
|
|
1055
|
+
return Target(self, target_info['targetId'])
|
|
1056
|
+
|
|
1057
|
+
async def must_get_current_page(self) -> 'Page':
|
|
1058
|
+
"""Get the current page as an actor Page."""
|
|
1059
|
+
page = await self.get_current_page()
|
|
1060
|
+
if not page:
|
|
1061
|
+
raise RuntimeError('No current target found')
|
|
1062
|
+
|
|
1063
|
+
return page
|
|
1064
|
+
|
|
1065
|
+
async def get_pages(self) -> list['Page']:
|
|
1066
|
+
"""Get all available pages."""
|
|
1067
|
+
result = await self.cdp_client.send.Target.getTargets()
|
|
1068
|
+
|
|
1069
|
+
targets = []
|
|
1070
|
+
# Import here to avoid circular import
|
|
1071
|
+
from browser_use.actor.page import Page as Target
|
|
1072
|
+
|
|
1073
|
+
for target_info in result['targetInfos']:
|
|
1074
|
+
if target_info['type'] in ['page', 'iframe']:
|
|
1075
|
+
targets.append(Target(self, target_info['targetId']))
|
|
1076
|
+
|
|
1077
|
+
return targets
|
|
1078
|
+
|
|
1079
|
+
async def close_page(self, page: 'Union[Page, str]') -> None:
|
|
1080
|
+
"""Close a page by Page object or target ID."""
|
|
1081
|
+
from cdp_use.cdp.target.commands import CloseTargetParameters
|
|
1082
|
+
|
|
1083
|
+
# Import here to avoid circular import
|
|
1084
|
+
from browser_use.actor.page import Page as Target
|
|
1085
|
+
|
|
1086
|
+
if isinstance(page, Target):
|
|
1087
|
+
target_id = page._target_id
|
|
1088
|
+
else:
|
|
1089
|
+
target_id = str(page)
|
|
1090
|
+
|
|
1091
|
+
params: CloseTargetParameters = {'targetId': target_id}
|
|
1092
|
+
await self.cdp_client.send.Target.closeTarget(params)
|
|
1093
|
+
|
|
1094
|
+
async def cookies(self, urls: list[str] | None = None) -> list['Cookie']:
|
|
1095
|
+
"""Get cookies, optionally filtered by URLs."""
|
|
1096
|
+
from cdp_use.cdp.network.library import GetCookiesParameters
|
|
1097
|
+
|
|
1098
|
+
params: GetCookiesParameters = {}
|
|
1099
|
+
if urls:
|
|
1100
|
+
params['urls'] = urls
|
|
1101
|
+
|
|
1102
|
+
result = await self.cdp_client.send.Network.getCookies(params)
|
|
1103
|
+
return result['cookies']
|
|
1104
|
+
|
|
1105
|
+
async def clear_cookies(self) -> None:
|
|
1106
|
+
"""Clear all cookies."""
|
|
1107
|
+
await self.cdp_client.send.Network.clearBrowserCookies()
|
|
1108
|
+
|
|
1109
|
+
async def export_storage_state(self, output_path: str | Path | None = None) -> dict[str, Any]:
|
|
1110
|
+
"""Export all browser cookies and storage to storage_state format.
|
|
1111
|
+
|
|
1112
|
+
Extracts decrypted cookies via CDP, bypassing keychain encryption.
|
|
1113
|
+
|
|
1114
|
+
Args:
|
|
1115
|
+
output_path: Optional path to save storage_state.json. If None, returns dict only.
|
|
1116
|
+
|
|
1117
|
+
Returns:
|
|
1118
|
+
Storage state dict with cookies in Playwright format.
|
|
1119
|
+
|
|
1120
|
+
"""
|
|
1121
|
+
from pathlib import Path
|
|
1122
|
+
|
|
1123
|
+
# Get all cookies using Storage.getCookies (returns decrypted cookies from all domains)
|
|
1124
|
+
cookies = await self._cdp_get_cookies()
|
|
1125
|
+
|
|
1126
|
+
# Convert CDP cookie format to Playwright storage_state format
|
|
1127
|
+
storage_state = {
|
|
1128
|
+
'cookies': [
|
|
1129
|
+
{
|
|
1130
|
+
'name': c['name'],
|
|
1131
|
+
'value': c['value'],
|
|
1132
|
+
'domain': c['domain'],
|
|
1133
|
+
'path': c['path'],
|
|
1134
|
+
'expires': c.get('expires', -1),
|
|
1135
|
+
'httpOnly': c.get('httpOnly', False),
|
|
1136
|
+
'secure': c.get('secure', False),
|
|
1137
|
+
'sameSite': c.get('sameSite', 'Lax'),
|
|
1138
|
+
}
|
|
1139
|
+
for c in cookies
|
|
1140
|
+
],
|
|
1141
|
+
'origins': [], # Could add localStorage/sessionStorage extraction if needed
|
|
1142
|
+
}
|
|
1143
|
+
|
|
1144
|
+
if output_path:
|
|
1145
|
+
import json
|
|
1146
|
+
|
|
1147
|
+
output_file = Path(output_path).expanduser().resolve()
|
|
1148
|
+
output_file.parent.mkdir(parents=True, exist_ok=True)
|
|
1149
|
+
output_file.write_text(json.dumps(storage_state, indent=2))
|
|
1150
|
+
self.logger.info(f'💾 Exported {len(cookies)} cookies to {output_file}')
|
|
1151
|
+
|
|
1152
|
+
return storage_state
|
|
1153
|
+
|
|
1154
|
+
async def get_or_create_cdp_session(self, target_id: TargetID | None = None, focus: bool = True) -> CDPSession:
|
|
1155
|
+
"""Get CDP session for a target from the event-driven pool.
|
|
1156
|
+
|
|
1157
|
+
With autoAttach=True, sessions are created automatically by Chrome and added
|
|
1158
|
+
to the pool via Target.attachedToTarget events. This method retrieves them.
|
|
1159
|
+
|
|
1160
|
+
Args:
|
|
1161
|
+
target_id: Target ID to get session for. If None, uses current agent focus.
|
|
1162
|
+
focus: If True, switches agent focus to this target.
|
|
1163
|
+
|
|
1164
|
+
Returns:
|
|
1165
|
+
CDPSession for the specified target.
|
|
1166
|
+
|
|
1167
|
+
Raises:
|
|
1168
|
+
ValueError: If target doesn't exist or session is not available.
|
|
1169
|
+
"""
|
|
1170
|
+
assert self._cdp_client_root is not None, 'Root CDP client not initialized'
|
|
1171
|
+
assert self.agent_focus is not None, 'CDP session not initialized'
|
|
1172
|
+
assert self._session_manager is not None, 'SessionManager not initialized'
|
|
1173
|
+
|
|
1174
|
+
# If no target_id specified, use current agent focus
|
|
1175
|
+
if target_id is None:
|
|
1176
|
+
target_id = self.agent_focus.target_id
|
|
1177
|
+
|
|
1178
|
+
# Get session from event-driven pool
|
|
1179
|
+
session = await self._session_manager.get_session_for_target(target_id)
|
|
1180
|
+
|
|
1181
|
+
if not session:
|
|
1182
|
+
# Session not in pool yet - wait for attach event
|
|
1183
|
+
self.logger.debug(f'[SessionManager] Waiting for target {target_id[:8]}... to attach...')
|
|
1184
|
+
|
|
1185
|
+
# Wait up to 2 seconds for the attach event
|
|
1186
|
+
for attempt in range(20):
|
|
1187
|
+
await asyncio.sleep(0.1)
|
|
1188
|
+
session = await self._session_manager.get_session_for_target(target_id)
|
|
1189
|
+
if session:
|
|
1190
|
+
self.logger.debug(f'[SessionManager] Target appeared after {attempt * 100}ms')
|
|
1191
|
+
break
|
|
1192
|
+
|
|
1193
|
+
if not session:
|
|
1194
|
+
# Timeout - target doesn't exist
|
|
1195
|
+
raise ValueError(f'Target {target_id} not found - may have detached or never existed')
|
|
1196
|
+
|
|
1197
|
+
# Validate session is still active
|
|
1198
|
+
is_valid = await self._session_manager.validate_session(target_id)
|
|
1199
|
+
if not is_valid:
|
|
1200
|
+
raise ValueError(f'Target {target_id} has detached - no active sessions')
|
|
1201
|
+
|
|
1202
|
+
# Update focus if requested
|
|
1203
|
+
# CRITICAL: Only allow focus change to 'page' type targets, not iframes/workers
|
|
1204
|
+
if focus and self.agent_focus.target_id != target_id:
|
|
1205
|
+
# Check target type before allowing focus change
|
|
1206
|
+
targets = await self._cdp_client_root.send.Target.getTargets()
|
|
1207
|
+
target_info = next((t for t in targets['targetInfos'] if t['targetId'] == target_id), None)
|
|
1208
|
+
target_type = target_info.get('type') if target_info else 'unknown'
|
|
1209
|
+
|
|
1210
|
+
if target_type == 'page':
|
|
1211
|
+
self.logger.debug(f'[SessionManager] Switching focus: {self.agent_focus.target_id[:8]}... → {target_id[:8]}...')
|
|
1212
|
+
self.agent_focus = session
|
|
1213
|
+
else:
|
|
1214
|
+
# Ignore focus request for non-page targets (iframes, workers, etc.)
|
|
1215
|
+
# These can detach at any time, causing agent_focus to point to dead target
|
|
1216
|
+
self.logger.debug(
|
|
1217
|
+
f'[SessionManager] Ignoring focus request for {target_type} target {target_id[:8]}... '
|
|
1218
|
+
f'(agent_focus stays on {self.agent_focus.target_id[:8]}...)'
|
|
1219
|
+
)
|
|
1220
|
+
|
|
1221
|
+
# Resume if waiting for debugger
|
|
1222
|
+
if focus:
|
|
1223
|
+
try:
|
|
1224
|
+
await session.cdp_client.send.Runtime.runIfWaitingForDebugger(session_id=session.session_id)
|
|
1225
|
+
except Exception:
|
|
1226
|
+
pass # May fail if not waiting
|
|
1227
|
+
|
|
1228
|
+
return session
|
|
1229
|
+
|
|
1230
|
+
@property
|
|
1231
|
+
def current_target_id(self) -> str | None:
|
|
1232
|
+
return self.agent_focus.target_id if self.agent_focus else None
|
|
1233
|
+
|
|
1234
|
+
@property
|
|
1235
|
+
def current_session_id(self) -> str | None:
|
|
1236
|
+
return self.agent_focus.session_id if self.agent_focus else None
|
|
1237
|
+
|
|
1238
|
+
# endregion - ========== CDP-based ... ==========
|
|
1239
|
+
|
|
1240
|
+
# region - ========== Helper Methods ==========
|
|
1241
|
+
@observe_debug(ignore_input=True, ignore_output=True, name='get_browser_state_summary')
|
|
1242
|
+
async def get_browser_state_summary(
|
|
1243
|
+
self,
|
|
1244
|
+
include_screenshot: bool = True,
|
|
1245
|
+
cached: bool = False,
|
|
1246
|
+
include_recent_events: bool = False,
|
|
1247
|
+
) -> BrowserStateSummary:
|
|
1248
|
+
if cached and self._cached_browser_state_summary is not None and self._cached_browser_state_summary.dom_state:
|
|
1249
|
+
# Don't use cached state if it has 0 interactive elements
|
|
1250
|
+
selector_map = self._cached_browser_state_summary.dom_state.selector_map
|
|
1251
|
+
|
|
1252
|
+
# Don't use cached state if we need a screenshot but the cached state doesn't have one
|
|
1253
|
+
if include_screenshot and not self._cached_browser_state_summary.screenshot:
|
|
1254
|
+
self.logger.debug('⚠️ Cached browser state has no screenshot, fetching fresh state with screenshot')
|
|
1255
|
+
# Fall through to fetch fresh state with screenshot
|
|
1256
|
+
elif selector_map and len(selector_map) > 0:
|
|
1257
|
+
self.logger.debug('🔄 Using pre-cached browser state summary for open tab')
|
|
1258
|
+
return self._cached_browser_state_summary
|
|
1259
|
+
else:
|
|
1260
|
+
self.logger.debug('⚠️ Cached browser state has 0 interactive elements, fetching fresh state')
|
|
1261
|
+
# Fall through to fetch fresh state
|
|
1262
|
+
|
|
1263
|
+
# Dispatch the event and wait for result
|
|
1264
|
+
event: BrowserStateRequestEvent = cast(
|
|
1265
|
+
BrowserStateRequestEvent,
|
|
1266
|
+
self.event_bus.dispatch(
|
|
1267
|
+
BrowserStateRequestEvent(
|
|
1268
|
+
include_dom=True,
|
|
1269
|
+
include_screenshot=include_screenshot,
|
|
1270
|
+
include_recent_events=include_recent_events,
|
|
1271
|
+
)
|
|
1272
|
+
),
|
|
1273
|
+
)
|
|
1274
|
+
|
|
1275
|
+
# The handler returns the BrowserStateSummary directly
|
|
1276
|
+
result = await event.event_result(raise_if_none=True, raise_if_any=True)
|
|
1277
|
+
assert result is not None and result.dom_state is not None
|
|
1278
|
+
return result
|
|
1279
|
+
|
|
1280
|
+
async def get_state_as_text(self) -> str:
|
|
1281
|
+
"""Get the browser state as text."""
|
|
1282
|
+
state = await self.get_browser_state_summary()
|
|
1283
|
+
assert state.dom_state is not None
|
|
1284
|
+
dom_state = state.dom_state
|
|
1285
|
+
return dom_state.llm_representation()
|
|
1286
|
+
|
|
1287
|
+
async def attach_all_watchdogs(self) -> None:
|
|
1288
|
+
"""Initialize and attach all watchdogs with explicit handler registration."""
|
|
1289
|
+
# Prevent duplicate watchdog attachment
|
|
1290
|
+
if hasattr(self, '_watchdogs_attached') and self._watchdogs_attached:
|
|
1291
|
+
self.logger.debug('Watchdogs already attached, skipping duplicate attachment')
|
|
1292
|
+
return
|
|
1293
|
+
|
|
1294
|
+
from browser_use.browser.watchdogs.aboutblank_watchdog import AboutBlankWatchdog
|
|
1295
|
+
|
|
1296
|
+
# from browser_use.browser.crash_watchdog import CrashWatchdog
|
|
1297
|
+
from browser_use.browser.watchdogs.default_action_watchdog import DefaultActionWatchdog
|
|
1298
|
+
from browser_use.browser.watchdogs.dom_watchdog import DOMWatchdog
|
|
1299
|
+
from browser_use.browser.watchdogs.downloads_watchdog import DownloadsWatchdog
|
|
1300
|
+
from browser_use.browser.watchdogs.local_browser_watchdog import LocalBrowserWatchdog
|
|
1301
|
+
from browser_use.browser.watchdogs.permissions_watchdog import PermissionsWatchdog
|
|
1302
|
+
from browser_use.browser.watchdogs.popups_watchdog import PopupsWatchdog
|
|
1303
|
+
from browser_use.browser.watchdogs.recording_watchdog import RecordingWatchdog
|
|
1304
|
+
from browser_use.browser.watchdogs.screenshot_watchdog import ScreenshotWatchdog
|
|
1305
|
+
from browser_use.browser.watchdogs.security_watchdog import SecurityWatchdog
|
|
1306
|
+
from browser_use.browser.watchdogs.storage_state_watchdog import StorageStateWatchdog
|
|
1307
|
+
|
|
1308
|
+
# Initialize CrashWatchdog
|
|
1309
|
+
# CrashWatchdog.model_rebuild()
|
|
1310
|
+
# self._crash_watchdog = CrashWatchdog(event_bus=self.event_bus, browser_session=self)
|
|
1311
|
+
# self.event_bus.on(BrowserConnectedEvent, self._crash_watchdog.on_BrowserConnectedEvent)
|
|
1312
|
+
# self.event_bus.on(BrowserStoppedEvent, self._crash_watchdog.on_BrowserStoppedEvent)
|
|
1313
|
+
# self._crash_watchdog.attach_to_session()
|
|
1314
|
+
|
|
1315
|
+
# Initialize DownloadsWatchdog
|
|
1316
|
+
DownloadsWatchdog.model_rebuild()
|
|
1317
|
+
self._downloads_watchdog = DownloadsWatchdog(event_bus=self.event_bus, browser_session=self)
|
|
1318
|
+
# self.event_bus.on(BrowserLaunchEvent, self._downloads_watchdog.on_BrowserLaunchEvent)
|
|
1319
|
+
# self.event_bus.on(TabCreatedEvent, self._downloads_watchdog.on_TabCreatedEvent)
|
|
1320
|
+
# self.event_bus.on(TabClosedEvent, self._downloads_watchdog.on_TabClosedEvent)
|
|
1321
|
+
# self.event_bus.on(BrowserStoppedEvent, self._downloads_watchdog.on_BrowserStoppedEvent)
|
|
1322
|
+
# self.event_bus.on(NavigationCompleteEvent, self._downloads_watchdog.on_NavigationCompleteEvent)
|
|
1323
|
+
self._downloads_watchdog.attach_to_session()
|
|
1324
|
+
if self.browser_profile.auto_download_pdfs:
|
|
1325
|
+
self.logger.debug('📄 PDF auto-download enabled for this session')
|
|
1326
|
+
|
|
1327
|
+
# Initialize StorageStateWatchdog conditionally
|
|
1328
|
+
# Enable when user provides either storage_state or user_data_dir (indicating they want persistence)
|
|
1329
|
+
should_enable_storage_state = (
|
|
1330
|
+
self.browser_profile.storage_state is not None or self.browser_profile.user_data_dir is not None
|
|
1331
|
+
)
|
|
1332
|
+
|
|
1333
|
+
if should_enable_storage_state:
|
|
1334
|
+
StorageStateWatchdog.model_rebuild()
|
|
1335
|
+
self._storage_state_watchdog = StorageStateWatchdog(
|
|
1336
|
+
event_bus=self.event_bus,
|
|
1337
|
+
browser_session=self,
|
|
1338
|
+
# More conservative defaults when auto-enabled
|
|
1339
|
+
auto_save_interval=60.0, # 1 minute instead of 30 seconds
|
|
1340
|
+
save_on_change=False, # Only save on shutdown by default
|
|
1341
|
+
)
|
|
1342
|
+
self._storage_state_watchdog.attach_to_session()
|
|
1343
|
+
self.logger.debug(
|
|
1344
|
+
f'🍪 StorageStateWatchdog enabled (storage_state: {bool(self.browser_profile.storage_state)}, user_data_dir: {bool(self.browser_profile.user_data_dir)})'
|
|
1345
|
+
)
|
|
1346
|
+
else:
|
|
1347
|
+
self.logger.debug('🍪 StorageStateWatchdog disabled (no storage_state or user_data_dir configured)')
|
|
1348
|
+
|
|
1349
|
+
# Initialize LocalBrowserWatchdog
|
|
1350
|
+
LocalBrowserWatchdog.model_rebuild()
|
|
1351
|
+
self._local_browser_watchdog = LocalBrowserWatchdog(event_bus=self.event_bus, browser_session=self)
|
|
1352
|
+
# self.event_bus.on(BrowserLaunchEvent, self._local_browser_watchdog.on_BrowserLaunchEvent)
|
|
1353
|
+
# self.event_bus.on(BrowserKillEvent, self._local_browser_watchdog.on_BrowserKillEvent)
|
|
1354
|
+
# self.event_bus.on(BrowserStopEvent, self._local_browser_watchdog.on_BrowserStopEvent)
|
|
1355
|
+
self._local_browser_watchdog.attach_to_session()
|
|
1356
|
+
|
|
1357
|
+
# Initialize SecurityWatchdog (hooks NavigationWatchdog and implements allowed_domains restriction)
|
|
1358
|
+
SecurityWatchdog.model_rebuild()
|
|
1359
|
+
self._security_watchdog = SecurityWatchdog(event_bus=self.event_bus, browser_session=self)
|
|
1360
|
+
# Core navigation is now handled in BrowserSession directly
|
|
1361
|
+
# SecurityWatchdog only handles security policy enforcement
|
|
1362
|
+
self._security_watchdog.attach_to_session()
|
|
1363
|
+
|
|
1364
|
+
# Initialize AboutBlankWatchdog (handles about:blank pages and DVD loading animation on first load)
|
|
1365
|
+
AboutBlankWatchdog.model_rebuild()
|
|
1366
|
+
self._aboutblank_watchdog = AboutBlankWatchdog(event_bus=self.event_bus, browser_session=self)
|
|
1367
|
+
# self.event_bus.on(BrowserStopEvent, self._aboutblank_watchdog.on_BrowserStopEvent)
|
|
1368
|
+
# self.event_bus.on(BrowserStoppedEvent, self._aboutblank_watchdog.on_BrowserStoppedEvent)
|
|
1369
|
+
# self.event_bus.on(TabCreatedEvent, self._aboutblank_watchdog.on_TabCreatedEvent)
|
|
1370
|
+
# self.event_bus.on(TabClosedEvent, self._aboutblank_watchdog.on_TabClosedEvent)
|
|
1371
|
+
self._aboutblank_watchdog.attach_to_session()
|
|
1372
|
+
|
|
1373
|
+
# Initialize PopupsWatchdog (handles accepting and dismissing JS dialogs, alerts, confirm, onbeforeunload, etc.)
|
|
1374
|
+
PopupsWatchdog.model_rebuild()
|
|
1375
|
+
self._popups_watchdog = PopupsWatchdog(event_bus=self.event_bus, browser_session=self)
|
|
1376
|
+
# self.event_bus.on(TabCreatedEvent, self._popups_watchdog.on_TabCreatedEvent)
|
|
1377
|
+
# self.event_bus.on(DialogCloseEvent, self._popups_watchdog.on_DialogCloseEvent)
|
|
1378
|
+
self._popups_watchdog.attach_to_session()
|
|
1379
|
+
|
|
1380
|
+
# Initialize PermissionsWatchdog (handles granting and revoking browser permissions like clipboard, microphone, camera, etc.)
|
|
1381
|
+
PermissionsWatchdog.model_rebuild()
|
|
1382
|
+
self._permissions_watchdog = PermissionsWatchdog(event_bus=self.event_bus, browser_session=self)
|
|
1383
|
+
# self.event_bus.on(BrowserConnectedEvent, self._permissions_watchdog.on_BrowserConnectedEvent)
|
|
1384
|
+
self._permissions_watchdog.attach_to_session()
|
|
1385
|
+
|
|
1386
|
+
# Initialize DefaultActionWatchdog (handles all default actions like click, type, scroll, go back, go forward, refresh, wait, send keys, upload file, scroll to text, etc.)
|
|
1387
|
+
DefaultActionWatchdog.model_rebuild()
|
|
1388
|
+
self._default_action_watchdog = DefaultActionWatchdog(event_bus=self.event_bus, browser_session=self)
|
|
1389
|
+
# self.event_bus.on(ClickElementEvent, self._default_action_watchdog.on_ClickElementEvent)
|
|
1390
|
+
# self.event_bus.on(TypeTextEvent, self._default_action_watchdog.on_TypeTextEvent)
|
|
1391
|
+
# self.event_bus.on(ScrollEvent, self._default_action_watchdog.on_ScrollEvent)
|
|
1392
|
+
# self.event_bus.on(GoBackEvent, self._default_action_watchdog.on_GoBackEvent)
|
|
1393
|
+
# self.event_bus.on(GoForwardEvent, self._default_action_watchdog.on_GoForwardEvent)
|
|
1394
|
+
# self.event_bus.on(RefreshEvent, self._default_action_watchdog.on_RefreshEvent)
|
|
1395
|
+
# self.event_bus.on(WaitEvent, self._default_action_watchdog.on_WaitEvent)
|
|
1396
|
+
# self.event_bus.on(SendKeysEvent, self._default_action_watchdog.on_SendKeysEvent)
|
|
1397
|
+
# self.event_bus.on(UploadFileEvent, self._default_action_watchdog.on_UploadFileEvent)
|
|
1398
|
+
# self.event_bus.on(ScrollToTextEvent, self._default_action_watchdog.on_ScrollToTextEvent)
|
|
1399
|
+
self._default_action_watchdog.attach_to_session()
|
|
1400
|
+
|
|
1401
|
+
# Initialize ScreenshotWatchdog (handles taking screenshots of the browser)
|
|
1402
|
+
ScreenshotWatchdog.model_rebuild()
|
|
1403
|
+
self._screenshot_watchdog = ScreenshotWatchdog(event_bus=self.event_bus, browser_session=self)
|
|
1404
|
+
# self.event_bus.on(BrowserStartEvent, self._screenshot_watchdog.on_BrowserStartEvent)
|
|
1405
|
+
# self.event_bus.on(BrowserStoppedEvent, self._screenshot_watchdog.on_BrowserStoppedEvent)
|
|
1406
|
+
# self.event_bus.on(ScreenshotEvent, self._screenshot_watchdog.on_ScreenshotEvent)
|
|
1407
|
+
self._screenshot_watchdog.attach_to_session()
|
|
1408
|
+
|
|
1409
|
+
# Initialize DOMWatchdog (handles building the DOM tree and detecting interactive elements, depends on ScreenshotWatchdog)
|
|
1410
|
+
DOMWatchdog.model_rebuild()
|
|
1411
|
+
self._dom_watchdog = DOMWatchdog(event_bus=self.event_bus, browser_session=self)
|
|
1412
|
+
# self.event_bus.on(TabCreatedEvent, self._dom_watchdog.on_TabCreatedEvent)
|
|
1413
|
+
# self.event_bus.on(BrowserStateRequestEvent, self._dom_watchdog.on_BrowserStateRequestEvent)
|
|
1414
|
+
self._dom_watchdog.attach_to_session()
|
|
1415
|
+
|
|
1416
|
+
# Initialize RecordingWatchdog (handles video recording)
|
|
1417
|
+
RecordingWatchdog.model_rebuild()
|
|
1418
|
+
self._recording_watchdog = RecordingWatchdog(event_bus=self.event_bus, browser_session=self)
|
|
1419
|
+
self._recording_watchdog.attach_to_session()
|
|
1420
|
+
|
|
1421
|
+
# Mark watchdogs as attached to prevent duplicate attachment
|
|
1422
|
+
self._watchdogs_attached = True
|
|
1423
|
+
|
|
1424
|
+
async def connect(self, cdp_url: str | None = None) -> Self:
|
|
1425
|
+
"""Connect to a remote chromium-based browser via CDP using cdp-use.
|
|
1426
|
+
|
|
1427
|
+
This MUST succeed or the browser is unusable. Fails hard on any error.
|
|
1428
|
+
"""
|
|
1429
|
+
|
|
1430
|
+
self.browser_profile.cdp_url = cdp_url or self.cdp_url
|
|
1431
|
+
if not self.cdp_url:
|
|
1432
|
+
raise RuntimeError('Cannot setup CDP connection without CDP URL')
|
|
1433
|
+
|
|
1434
|
+
if not self.cdp_url.startswith('ws'):
|
|
1435
|
+
# If it's an HTTP URL, fetch the WebSocket URL from /json/version endpoint
|
|
1436
|
+
url = self.cdp_url.rstrip('/')
|
|
1437
|
+
if not url.endswith('/json/version'):
|
|
1438
|
+
url = url + '/json/version'
|
|
1439
|
+
|
|
1440
|
+
# Run a tiny HTTP client to query for the WebSocket URL from the /json/version endpoint
|
|
1441
|
+
async with httpx.AsyncClient() as client:
|
|
1442
|
+
headers = self.browser_profile.headers or {}
|
|
1443
|
+
version_info = await client.get(url, headers=headers)
|
|
1444
|
+
self.browser_profile.cdp_url = version_info.json()['webSocketDebuggerUrl']
|
|
1445
|
+
|
|
1446
|
+
assert self.cdp_url is not None
|
|
1447
|
+
|
|
1448
|
+
browser_location = 'local browser' if self.is_local else 'remote browser'
|
|
1449
|
+
self.logger.debug(f'🌎 Connecting to existing chromium-based browser via CDP: {self.cdp_url} -> ({browser_location})')
|
|
1450
|
+
|
|
1451
|
+
try:
|
|
1452
|
+
# Create and store the CDP client for direct CDP communication
|
|
1453
|
+
self._cdp_client_root = CDPClient(self.cdp_url)
|
|
1454
|
+
assert self._cdp_client_root is not None
|
|
1455
|
+
await self._cdp_client_root.start()
|
|
1456
|
+
|
|
1457
|
+
# Initialize event-driven session manager FIRST (before enabling autoAttach)
|
|
1458
|
+
from browser_use.browser.session_manager import SessionManager
|
|
1459
|
+
|
|
1460
|
+
self._session_manager = SessionManager(self)
|
|
1461
|
+
await self._session_manager.start_monitoring()
|
|
1462
|
+
self.logger.debug('Event-driven session manager started')
|
|
1463
|
+
|
|
1464
|
+
# Enable auto-attach so Chrome automatically notifies us when NEW targets attach/detach
|
|
1465
|
+
# This is the foundation of event-driven session management
|
|
1466
|
+
await self._cdp_client_root.send.Target.setAutoAttach(
|
|
1467
|
+
params={'autoAttach': True, 'waitForDebuggerOnStart': False, 'flatten': True}
|
|
1468
|
+
)
|
|
1469
|
+
self.logger.debug('CDP client connected with auto-attach enabled')
|
|
1470
|
+
|
|
1471
|
+
# Get browser targets to find available contexts/pages
|
|
1472
|
+
targets = await self._cdp_client_root.send.Target.getTargets()
|
|
1473
|
+
|
|
1474
|
+
# Manually attach to ALL EXISTING targets (autoAttach only fires for new ones)
|
|
1475
|
+
# We attach to everything (pages, iframes, workers) for complete coverage
|
|
1476
|
+
for target in targets['targetInfos']:
|
|
1477
|
+
target_id = target['targetId']
|
|
1478
|
+
target_type = target.get('type', 'unknown')
|
|
1479
|
+
|
|
1480
|
+
try:
|
|
1481
|
+
# Attach to target - this triggers attachedToTarget event
|
|
1482
|
+
result = await self._cdp_client_root.send.Target.attachToTarget(
|
|
1483
|
+
params={'targetId': target_id, 'flatten': True}
|
|
1484
|
+
)
|
|
1485
|
+
session_id = result['sessionId']
|
|
1486
|
+
|
|
1487
|
+
# Enable auto-attach for this target's children
|
|
1488
|
+
await self._cdp_client_root.send.Target.setAutoAttach(
|
|
1489
|
+
params={'autoAttach': True, 'waitForDebuggerOnStart': False, 'flatten': True}, session_id=session_id
|
|
1490
|
+
)
|
|
1491
|
+
|
|
1492
|
+
self.logger.debug(
|
|
1493
|
+
f'Attached to existing target: {target_id[:8]}... (type={target_type}, session={session_id[:8]}...)'
|
|
1494
|
+
)
|
|
1495
|
+
except Exception as e:
|
|
1496
|
+
self.logger.debug(f'Failed to attach to existing target {target_id[:8]}... (type={target_type}): {e}')
|
|
1497
|
+
|
|
1498
|
+
# Find main browser pages (avoiding iframes, workers, extensions, etc.)
|
|
1499
|
+
page_targets: list[TargetInfo] = [
|
|
1500
|
+
t
|
|
1501
|
+
for t in targets['targetInfos']
|
|
1502
|
+
if self._is_valid_target(
|
|
1503
|
+
t, include_http=True, include_about=True, include_pages=True, include_iframes=False, include_workers=False
|
|
1504
|
+
)
|
|
1505
|
+
]
|
|
1506
|
+
|
|
1507
|
+
# Check for chrome://newtab pages and redirect them to about:blank
|
|
1508
|
+
from browser_use.utils import is_new_tab_page
|
|
1509
|
+
|
|
1510
|
+
for target in page_targets:
|
|
1511
|
+
target_url = target.get('url', '')
|
|
1512
|
+
if is_new_tab_page(target_url) and target_url != 'about:blank':
|
|
1513
|
+
target_id = target['targetId']
|
|
1514
|
+
self.logger.debug(f'🔄 Redirecting {target_url} to about:blank for target {target_id}')
|
|
1515
|
+
try:
|
|
1516
|
+
# Sessions now exist from manual attachment above
|
|
1517
|
+
session = await self._session_manager.get_session_for_target(target_id)
|
|
1518
|
+
if session:
|
|
1519
|
+
await session.cdp_client.send.Page.navigate(
|
|
1520
|
+
params={'url': 'about:blank'}, session_id=session.session_id
|
|
1521
|
+
)
|
|
1522
|
+
target['url'] = 'about:blank'
|
|
1523
|
+
await asyncio.sleep(0.05) # Let navigation start
|
|
1524
|
+
except Exception as e:
|
|
1525
|
+
self.logger.warning(f'Failed to redirect {target_url}: {e}')
|
|
1526
|
+
|
|
1527
|
+
# Ensure we have at least one page
|
|
1528
|
+
if not page_targets:
|
|
1529
|
+
new_target = await self._cdp_client_root.send.Target.createTarget(params={'url': 'about:blank'})
|
|
1530
|
+
target_id = new_target['targetId']
|
|
1531
|
+
self.logger.debug(f'📄 Created new blank page: {target_id}')
|
|
1532
|
+
else:
|
|
1533
|
+
target_id = [page for page in page_targets if page.get('type') == 'page'][0]['targetId']
|
|
1534
|
+
self.logger.debug(f'📄 Using existing page: {target_id}')
|
|
1535
|
+
|
|
1536
|
+
# Wait for SessionManager to receive the attach event for this target
|
|
1537
|
+
# (Chrome will fire Target.attachedToTarget event which SessionManager handles)
|
|
1538
|
+
for _ in range(20): # Wait up to 2 seconds
|
|
1539
|
+
await asyncio.sleep(0.1)
|
|
1540
|
+
session = await self._session_manager.get_session_for_target(target_id)
|
|
1541
|
+
if session:
|
|
1542
|
+
self.agent_focus = session
|
|
1543
|
+
# SessionManager already added it to pool - no need to do it manually
|
|
1544
|
+
self.logger.debug(f'📄 Agent focus set to {target_id[:8]}...')
|
|
1545
|
+
break
|
|
1546
|
+
|
|
1547
|
+
if not self.agent_focus:
|
|
1548
|
+
raise RuntimeError(f'Failed to get session for initial target {target_id}')
|
|
1549
|
+
|
|
1550
|
+
# Enable proxy authentication handling if configured
|
|
1551
|
+
await self._setup_proxy_auth()
|
|
1552
|
+
|
|
1553
|
+
# Verify the session is working
|
|
1554
|
+
if self.agent_focus.title == 'Unknown title':
|
|
1555
|
+
self.logger.warning('Session created but title is unknown (may be normal for about:blank)')
|
|
1556
|
+
|
|
1557
|
+
# Dispatch TabCreatedEvent for all initial tabs (so watchdogs can initialize)
|
|
1558
|
+
for idx, target in enumerate(page_targets):
|
|
1559
|
+
target_url = target.get('url', '')
|
|
1560
|
+
self.logger.debug(f'Dispatching TabCreatedEvent for initial tab {idx}: {target_url}')
|
|
1561
|
+
self.event_bus.dispatch(TabCreatedEvent(url=target_url, target_id=target['targetId']))
|
|
1562
|
+
|
|
1563
|
+
# Dispatch initial focus event
|
|
1564
|
+
if page_targets:
|
|
1565
|
+
initial_url = page_targets[0].get('url', '')
|
|
1566
|
+
self.event_bus.dispatch(AgentFocusChangedEvent(target_id=page_targets[0]['targetId'], url=initial_url))
|
|
1567
|
+
self.logger.debug(f'Initial agent focus set to tab 0: {initial_url}')
|
|
1568
|
+
|
|
1569
|
+
except Exception as e:
|
|
1570
|
+
# Fatal error - browser is not usable without CDP connection
|
|
1571
|
+
self.logger.error(f'❌ FATAL: Failed to setup CDP connection: {e}')
|
|
1572
|
+
self.logger.error('❌ Browser cannot continue without CDP connection')
|
|
1573
|
+
# Clean up any partial state
|
|
1574
|
+
self._cdp_client_root = None
|
|
1575
|
+
self.agent_focus = None
|
|
1576
|
+
# Re-raise as a fatal error
|
|
1577
|
+
raise RuntimeError(f'Failed to establish CDP connection to browser: {e}') from e
|
|
1578
|
+
|
|
1579
|
+
return self
|
|
1580
|
+
|
|
1581
|
+
async def _setup_proxy_auth(self) -> None:
|
|
1582
|
+
"""Enable CDP Fetch auth handling for authenticated proxy, if credentials provided.
|
|
1583
|
+
|
|
1584
|
+
Handles HTTP proxy authentication challenges (Basic/Proxy) by providing
|
|
1585
|
+
configured credentials from BrowserProfile.
|
|
1586
|
+
"""
|
|
1587
|
+
|
|
1588
|
+
assert self._cdp_client_root
|
|
1589
|
+
|
|
1590
|
+
try:
|
|
1591
|
+
proxy_cfg = self.browser_profile.proxy
|
|
1592
|
+
username = proxy_cfg.username if proxy_cfg else None
|
|
1593
|
+
password = proxy_cfg.password if proxy_cfg else None
|
|
1594
|
+
if not username or not password:
|
|
1595
|
+
self.logger.debug('Proxy credentials not provided; skipping proxy auth setup')
|
|
1596
|
+
return
|
|
1597
|
+
|
|
1598
|
+
# Enable Fetch domain with auth handling (do not pause all requests)
|
|
1599
|
+
try:
|
|
1600
|
+
await self._cdp_client_root.send.Fetch.enable(params={'handleAuthRequests': True})
|
|
1601
|
+
self.logger.debug('Fetch.enable(handleAuthRequests=True) enabled on root client')
|
|
1602
|
+
except Exception as e:
|
|
1603
|
+
self.logger.debug(f'Fetch.enable on root failed: {type(e).__name__}: {e}')
|
|
1604
|
+
|
|
1605
|
+
# Also enable on the focused session if available to ensure events are delivered
|
|
1606
|
+
try:
|
|
1607
|
+
if self.agent_focus:
|
|
1608
|
+
await self.agent_focus.cdp_client.send.Fetch.enable(
|
|
1609
|
+
params={'handleAuthRequests': True},
|
|
1610
|
+
session_id=self.agent_focus.session_id,
|
|
1611
|
+
)
|
|
1612
|
+
self.logger.debug('Fetch.enable(handleAuthRequests=True) enabled on focused session')
|
|
1613
|
+
except Exception as e:
|
|
1614
|
+
self.logger.debug(f'Fetch.enable on focused session failed: {type(e).__name__}: {e}')
|
|
1615
|
+
|
|
1616
|
+
def _on_auth_required(event: AuthRequiredEvent, session_id: SessionID | None = None):
|
|
1617
|
+
# event keys may be snake_case or camelCase depending on generator; handle both
|
|
1618
|
+
request_id = event.get('requestId') or event.get('request_id')
|
|
1619
|
+
if not request_id:
|
|
1620
|
+
return
|
|
1621
|
+
|
|
1622
|
+
challenge = event.get('authChallenge') or event.get('auth_challenge') or {}
|
|
1623
|
+
source = (challenge.get('source') or '').lower()
|
|
1624
|
+
# Only respond to proxy challenges
|
|
1625
|
+
if source == 'proxy' and request_id:
|
|
1626
|
+
|
|
1627
|
+
async def _respond():
|
|
1628
|
+
assert self._cdp_client_root
|
|
1629
|
+
try:
|
|
1630
|
+
await self._cdp_client_root.send.Fetch.continueWithAuth(
|
|
1631
|
+
params={
|
|
1632
|
+
'requestId': request_id,
|
|
1633
|
+
'authChallengeResponse': {
|
|
1634
|
+
'response': 'ProvideCredentials',
|
|
1635
|
+
'username': username,
|
|
1636
|
+
'password': password,
|
|
1637
|
+
},
|
|
1638
|
+
},
|
|
1639
|
+
session_id=session_id,
|
|
1640
|
+
)
|
|
1641
|
+
except Exception as e:
|
|
1642
|
+
self.logger.debug(f'Proxy auth respond failed: {type(e).__name__}: {e}')
|
|
1643
|
+
|
|
1644
|
+
# schedule
|
|
1645
|
+
asyncio.create_task(_respond())
|
|
1646
|
+
else:
|
|
1647
|
+
# Default behaviour for non-proxy challenges: let browser handle
|
|
1648
|
+
async def _default():
|
|
1649
|
+
assert self._cdp_client_root
|
|
1650
|
+
try:
|
|
1651
|
+
await self._cdp_client_root.send.Fetch.continueWithAuth(
|
|
1652
|
+
params={'requestId': request_id, 'authChallengeResponse': {'response': 'Default'}},
|
|
1653
|
+
session_id=session_id,
|
|
1654
|
+
)
|
|
1655
|
+
except Exception as e:
|
|
1656
|
+
self.logger.debug(f'Default auth respond failed: {type(e).__name__}: {e}')
|
|
1657
|
+
|
|
1658
|
+
if request_id:
|
|
1659
|
+
asyncio.create_task(_default())
|
|
1660
|
+
|
|
1661
|
+
def _on_request_paused(event: RequestPausedEvent, session_id: SessionID | None = None):
|
|
1662
|
+
# Continue all paused requests to avoid stalling the network
|
|
1663
|
+
request_id = event.get('requestId') or event.get('request_id')
|
|
1664
|
+
if not request_id:
|
|
1665
|
+
return
|
|
1666
|
+
|
|
1667
|
+
async def _continue():
|
|
1668
|
+
assert self._cdp_client_root
|
|
1669
|
+
try:
|
|
1670
|
+
await self._cdp_client_root.send.Fetch.continueRequest(
|
|
1671
|
+
params={'requestId': request_id},
|
|
1672
|
+
session_id=session_id,
|
|
1673
|
+
)
|
|
1674
|
+
except Exception:
|
|
1675
|
+
pass
|
|
1676
|
+
|
|
1677
|
+
asyncio.create_task(_continue())
|
|
1678
|
+
|
|
1679
|
+
# Register event handler on root client
|
|
1680
|
+
try:
|
|
1681
|
+
self._cdp_client_root.register.Fetch.authRequired(_on_auth_required)
|
|
1682
|
+
self._cdp_client_root.register.Fetch.requestPaused(_on_request_paused)
|
|
1683
|
+
if self.agent_focus:
|
|
1684
|
+
self.agent_focus.cdp_client.register.Fetch.authRequired(_on_auth_required)
|
|
1685
|
+
self.agent_focus.cdp_client.register.Fetch.requestPaused(_on_request_paused)
|
|
1686
|
+
self.logger.debug('Registered Fetch.authRequired handlers')
|
|
1687
|
+
except Exception as e:
|
|
1688
|
+
self.logger.debug(f'Failed to register authRequired handlers: {type(e).__name__}: {e}')
|
|
1689
|
+
|
|
1690
|
+
# Auto-enable Fetch on every newly attached target to ensure auth callbacks fire
|
|
1691
|
+
def _on_attached(event: AttachedToTargetEvent, session_id: SessionID | None = None):
|
|
1692
|
+
sid = event.get('sessionId') or event.get('session_id') or session_id
|
|
1693
|
+
if not sid:
|
|
1694
|
+
return
|
|
1695
|
+
|
|
1696
|
+
async def _enable():
|
|
1697
|
+
assert self._cdp_client_root
|
|
1698
|
+
try:
|
|
1699
|
+
await self._cdp_client_root.send.Fetch.enable(
|
|
1700
|
+
params={'handleAuthRequests': True},
|
|
1701
|
+
session_id=sid,
|
|
1702
|
+
)
|
|
1703
|
+
self.logger.debug(f'Fetch.enable(handleAuthRequests=True) enabled on attached session {sid}')
|
|
1704
|
+
except Exception as e:
|
|
1705
|
+
self.logger.debug(f'Fetch.enable on attached session failed: {type(e).__name__}: {e}')
|
|
1706
|
+
|
|
1707
|
+
asyncio.create_task(_enable())
|
|
1708
|
+
|
|
1709
|
+
try:
|
|
1710
|
+
self._cdp_client_root.register.Target.attachedToTarget(_on_attached)
|
|
1711
|
+
self.logger.debug('Registered Target.attachedToTarget handler for Fetch.enable')
|
|
1712
|
+
except Exception as e:
|
|
1713
|
+
self.logger.debug(f'Failed to register attachedToTarget handler: {type(e).__name__}: {e}')
|
|
1714
|
+
|
|
1715
|
+
# Ensure Fetch is enabled for the current focused session, too
|
|
1716
|
+
try:
|
|
1717
|
+
if self.agent_focus:
|
|
1718
|
+
await self.agent_focus.cdp_client.send.Fetch.enable(
|
|
1719
|
+
params={'handleAuthRequests': True, 'patterns': [{'urlPattern': '*'}]},
|
|
1720
|
+
session_id=self.agent_focus.session_id,
|
|
1721
|
+
)
|
|
1722
|
+
except Exception as e:
|
|
1723
|
+
self.logger.debug(f'Fetch.enable on focused session failed: {type(e).__name__}: {e}')
|
|
1724
|
+
except Exception as e:
|
|
1725
|
+
self.logger.debug(f'Skipping proxy auth setup: {type(e).__name__}: {e}')
|
|
1726
|
+
|
|
1727
|
+
async def get_tabs(self) -> list[TabInfo]:
|
|
1728
|
+
"""Get information about all open tabs using CDP Target.getTargetInfo for speed."""
|
|
1729
|
+
tabs = []
|
|
1730
|
+
|
|
1731
|
+
# Safety check - return empty list if browser not connected yet
|
|
1732
|
+
if not self._cdp_client_root:
|
|
1733
|
+
return tabs
|
|
1734
|
+
|
|
1735
|
+
# Get all page targets using CDP
|
|
1736
|
+
pages = await self._cdp_get_all_pages()
|
|
1737
|
+
|
|
1738
|
+
for i, page_target in enumerate(pages):
|
|
1739
|
+
target_id = page_target['targetId']
|
|
1740
|
+
url = page_target['url']
|
|
1741
|
+
|
|
1742
|
+
# Try to get the title directly from Target.getTargetInfo - much faster!
|
|
1743
|
+
# The initial getTargets() doesn't include title, but getTargetInfo does
|
|
1744
|
+
try:
|
|
1745
|
+
target_info = await self.cdp_client.send.Target.getTargetInfo(params={'targetId': target_id})
|
|
1746
|
+
# The title is directly available in targetInfo
|
|
1747
|
+
title = target_info.get('targetInfo', {}).get('title', '')
|
|
1748
|
+
|
|
1749
|
+
# Skip JS execution for chrome:// pages and new tab pages
|
|
1750
|
+
if is_new_tab_page(url) or url.startswith('chrome://'):
|
|
1751
|
+
# Use URL as title for chrome pages, or mark new tabs as unusable
|
|
1752
|
+
if is_new_tab_page(url):
|
|
1753
|
+
title = ''
|
|
1754
|
+
elif not title:
|
|
1755
|
+
# For chrome:// pages without a title, use the URL itself
|
|
1756
|
+
title = url
|
|
1757
|
+
|
|
1758
|
+
# Special handling for PDF pages without titles
|
|
1759
|
+
if (not title or title == '') and (url.endswith('.pdf') or 'pdf' in url):
|
|
1760
|
+
# PDF pages might not have a title, use URL filename
|
|
1761
|
+
try:
|
|
1762
|
+
from urllib.parse import urlparse
|
|
1763
|
+
|
|
1764
|
+
filename = urlparse(url).path.split('/')[-1]
|
|
1765
|
+
if filename:
|
|
1766
|
+
title = filename
|
|
1767
|
+
except Exception:
|
|
1768
|
+
pass
|
|
1769
|
+
|
|
1770
|
+
except Exception as e:
|
|
1771
|
+
# Fallback to basic title handling
|
|
1772
|
+
self.logger.debug(f'⚠️ Failed to get target info for tab #{i}: {_log_pretty_url(url)} - {type(e).__name__}')
|
|
1773
|
+
|
|
1774
|
+
if is_new_tab_page(url):
|
|
1775
|
+
title = ''
|
|
1776
|
+
elif url.startswith('chrome://'):
|
|
1777
|
+
title = url
|
|
1778
|
+
else:
|
|
1779
|
+
title = ''
|
|
1780
|
+
|
|
1781
|
+
tab_info = TabInfo(
|
|
1782
|
+
target_id=target_id,
|
|
1783
|
+
url=url,
|
|
1784
|
+
title=title,
|
|
1785
|
+
parent_target_id=None,
|
|
1786
|
+
)
|
|
1787
|
+
tabs.append(tab_info)
|
|
1788
|
+
|
|
1789
|
+
return tabs
|
|
1790
|
+
|
|
1791
|
+
# endregion - ========== Helper Methods ==========
|
|
1792
|
+
|
|
1793
|
+
# region - ========== ID Lookup Methods ==========
|
|
1794
|
+
async def get_current_target_info(self) -> TargetInfo | None:
|
|
1795
|
+
"""Get info about the current active target using CDP."""
|
|
1796
|
+
if not self.agent_focus or not self.agent_focus.target_id:
|
|
1797
|
+
return None
|
|
1798
|
+
|
|
1799
|
+
targets = await self.cdp_client.send.Target.getTargets()
|
|
1800
|
+
for target in targets.get('targetInfos', []):
|
|
1801
|
+
if target.get('targetId') == self.agent_focus.target_id:
|
|
1802
|
+
# Still return even if it's not a "valid" target since we're looking for a specific ID
|
|
1803
|
+
return target
|
|
1804
|
+
return None
|
|
1805
|
+
|
|
1806
|
+
async def get_current_page_url(self) -> str:
|
|
1807
|
+
"""Get the URL of the current page using CDP."""
|
|
1808
|
+
target = await self.get_current_target_info()
|
|
1809
|
+
if target:
|
|
1810
|
+
return target.get('url', '')
|
|
1811
|
+
return 'about:blank'
|
|
1812
|
+
|
|
1813
|
+
async def get_current_page_title(self) -> str:
|
|
1814
|
+
"""Get the title of the current page using CDP."""
|
|
1815
|
+
target_info = await self.get_current_target_info()
|
|
1816
|
+
if target_info:
|
|
1817
|
+
return target_info.get('title', 'Unknown page title')
|
|
1818
|
+
return 'Unknown page title'
|
|
1819
|
+
|
|
1820
|
+
async def navigate_to(self, url: str, new_tab: bool = False) -> None:
|
|
1821
|
+
"""Navigate to a URL using the standard event system.
|
|
1822
|
+
|
|
1823
|
+
Args:
|
|
1824
|
+
url: URL to navigate to
|
|
1825
|
+
new_tab: Whether to open in a new tab
|
|
1826
|
+
"""
|
|
1827
|
+
from browser_use.browser.events import NavigateToUrlEvent
|
|
1828
|
+
|
|
1829
|
+
event = self.event_bus.dispatch(NavigateToUrlEvent(url=url, new_tab=new_tab))
|
|
1830
|
+
await event
|
|
1831
|
+
await event.event_result(raise_if_any=True, raise_if_none=False)
|
|
1832
|
+
|
|
1833
|
+
# endregion - ========== ID Lookup Methods ==========
|
|
1834
|
+
|
|
1835
|
+
# region - ========== DOM Helper Methods ==========
|
|
1836
|
+
|
|
1837
|
+
async def get_dom_element_by_index(self, index: int) -> EnhancedDOMTreeNode | None:
|
|
1838
|
+
"""Get DOM element by index.
|
|
1839
|
+
|
|
1840
|
+
Get element from cached selector map.
|
|
1841
|
+
|
|
1842
|
+
Args:
|
|
1843
|
+
index: The element index from the serialized DOM
|
|
1844
|
+
|
|
1845
|
+
Returns:
|
|
1846
|
+
EnhancedDOMTreeNode or None if index not found
|
|
1847
|
+
"""
|
|
1848
|
+
# Check cached selector map
|
|
1849
|
+
if self._cached_selector_map and index in self._cached_selector_map:
|
|
1850
|
+
return self._cached_selector_map[index]
|
|
1851
|
+
|
|
1852
|
+
return None
|
|
1853
|
+
|
|
1854
|
+
def update_cached_selector_map(self, selector_map: dict[int, EnhancedDOMTreeNode]) -> None:
|
|
1855
|
+
"""Update the cached selector map with new DOM state.
|
|
1856
|
+
|
|
1857
|
+
This should be called by the DOM watchdog after rebuilding the DOM.
|
|
1858
|
+
|
|
1859
|
+
Args:
|
|
1860
|
+
selector_map: The new selector map from DOM serialization
|
|
1861
|
+
"""
|
|
1862
|
+
self._cached_selector_map = selector_map
|
|
1863
|
+
|
|
1864
|
+
# Alias for backwards compatibility
|
|
1865
|
+
async def get_element_by_index(self, index: int) -> EnhancedDOMTreeNode | None:
|
|
1866
|
+
"""Alias for get_dom_element_by_index for backwards compatibility."""
|
|
1867
|
+
return await self.get_dom_element_by_index(index)
|
|
1868
|
+
|
|
1869
|
+
async def get_target_id_from_tab_id(self, tab_id: str) -> TargetID:
|
|
1870
|
+
"""Get the full-length TargetID from the truncated 4-char tab_id."""
|
|
1871
|
+
# First check cached sessions
|
|
1872
|
+
for full_target_id in self._cdp_session_pool.keys():
|
|
1873
|
+
if full_target_id.endswith(tab_id):
|
|
1874
|
+
if await self._is_target_valid(full_target_id):
|
|
1875
|
+
return full_target_id
|
|
1876
|
+
# Stale session - Chrome should have sent detach event
|
|
1877
|
+
# If we're here, event listener will clean it up
|
|
1878
|
+
self.logger.debug(f'Found stale session for target {full_target_id}, skipping')
|
|
1879
|
+
|
|
1880
|
+
# Get all current targets and find the one matching tab_id
|
|
1881
|
+
all_targets = await self.cdp_client.send.Target.getTargets()
|
|
1882
|
+
# Filter for valid page/tab targets only
|
|
1883
|
+
for target in all_targets.get('targetInfos', []):
|
|
1884
|
+
if target['targetId'].endswith(tab_id) and target.get('type') == 'page':
|
|
1885
|
+
return target['targetId']
|
|
1886
|
+
|
|
1887
|
+
raise ValueError(f'No TargetID found ending in tab_id=...{tab_id}')
|
|
1888
|
+
|
|
1889
|
+
async def _is_target_valid(self, target_id: TargetID) -> bool:
|
|
1890
|
+
"""Check if a target ID is still valid."""
|
|
1891
|
+
try:
|
|
1892
|
+
await self.cdp_client.send.Target.getTargetInfo(params={'targetId': target_id})
|
|
1893
|
+
return True
|
|
1894
|
+
except Exception:
|
|
1895
|
+
return False
|
|
1896
|
+
|
|
1897
|
+
async def get_target_id_from_url(self, url: str) -> TargetID:
|
|
1898
|
+
"""Get the TargetID from a URL."""
|
|
1899
|
+
all_targets = await self.cdp_client.send.Target.getTargets()
|
|
1900
|
+
for target in all_targets.get('targetInfos', []):
|
|
1901
|
+
if target['url'] == url and target['type'] == 'page':
|
|
1902
|
+
return target['targetId']
|
|
1903
|
+
|
|
1904
|
+
# still not found, try substring match as fallback
|
|
1905
|
+
for target in all_targets.get('targetInfos', []):
|
|
1906
|
+
if url in target['url'] and target['type'] == 'page':
|
|
1907
|
+
return target['targetId']
|
|
1908
|
+
|
|
1909
|
+
raise ValueError(f'No TargetID found for url={url}')
|
|
1910
|
+
|
|
1911
|
+
async def get_most_recently_opened_target_id(self) -> TargetID:
|
|
1912
|
+
"""Get the most recently opened target ID."""
|
|
1913
|
+
all_targets = await self.cdp_client.send.Target.getTargets()
|
|
1914
|
+
return (await self._cdp_get_all_pages())[-1]['targetId']
|
|
1915
|
+
|
|
1916
|
+
def is_file_input(self, element: Any) -> bool:
|
|
1917
|
+
"""Check if element is a file input.
|
|
1918
|
+
|
|
1919
|
+
Args:
|
|
1920
|
+
element: The DOM element to check
|
|
1921
|
+
|
|
1922
|
+
Returns:
|
|
1923
|
+
True if element is a file input, False otherwise
|
|
1924
|
+
"""
|
|
1925
|
+
if self._dom_watchdog:
|
|
1926
|
+
return self._dom_watchdog.is_file_input(element)
|
|
1927
|
+
# Fallback if watchdog not available
|
|
1928
|
+
return (
|
|
1929
|
+
hasattr(element, 'node_name')
|
|
1930
|
+
and element.node_name.upper() == 'INPUT'
|
|
1931
|
+
and hasattr(element, 'attributes')
|
|
1932
|
+
and element.attributes.get('type', '').lower() == 'file'
|
|
1933
|
+
)
|
|
1934
|
+
|
|
1935
|
+
async def get_selector_map(self) -> dict[int, EnhancedDOMTreeNode]:
|
|
1936
|
+
"""Get the current selector map from cached state or DOM watchdog.
|
|
1937
|
+
|
|
1938
|
+
Returns:
|
|
1939
|
+
Dictionary mapping element indices to EnhancedDOMTreeNode objects
|
|
1940
|
+
"""
|
|
1941
|
+
# First try cached selector map
|
|
1942
|
+
if self._cached_selector_map:
|
|
1943
|
+
return self._cached_selector_map
|
|
1944
|
+
|
|
1945
|
+
# Try to get from DOM watchdog
|
|
1946
|
+
if self._dom_watchdog and hasattr(self._dom_watchdog, 'selector_map'):
|
|
1947
|
+
return self._dom_watchdog.selector_map or {}
|
|
1948
|
+
|
|
1949
|
+
# Return empty dict if nothing available
|
|
1950
|
+
return {}
|
|
1951
|
+
|
|
1952
|
+
async def get_index_by_id(self, element_id: str) -> int | None:
|
|
1953
|
+
"""Find element index by its id attribute.
|
|
1954
|
+
|
|
1955
|
+
Args:
|
|
1956
|
+
element_id: The id attribute value to search for
|
|
1957
|
+
|
|
1958
|
+
Returns:
|
|
1959
|
+
Index of the element, or None if not found
|
|
1960
|
+
"""
|
|
1961
|
+
selector_map = await self.get_selector_map()
|
|
1962
|
+
for idx, element in selector_map.items():
|
|
1963
|
+
if element.attributes and element.attributes.get('id') == element_id:
|
|
1964
|
+
return idx
|
|
1965
|
+
return None
|
|
1966
|
+
|
|
1967
|
+
async def get_index_by_class(self, class_name: str) -> int | None:
|
|
1968
|
+
"""Find element index by its class attribute (matches if class contains the given name).
|
|
1969
|
+
|
|
1970
|
+
Args:
|
|
1971
|
+
class_name: The class name to search for
|
|
1972
|
+
|
|
1973
|
+
Returns:
|
|
1974
|
+
Index of the first matching element, or None if not found
|
|
1975
|
+
"""
|
|
1976
|
+
selector_map = await self.get_selector_map()
|
|
1977
|
+
for idx, element in selector_map.items():
|
|
1978
|
+
if element.attributes:
|
|
1979
|
+
element_class = element.attributes.get('class', '')
|
|
1980
|
+
if class_name in element_class.split():
|
|
1981
|
+
return idx
|
|
1982
|
+
return None
|
|
1983
|
+
|
|
1984
|
+
async def remove_highlights(self) -> None:
|
|
1985
|
+
"""Remove highlights from the page using CDP."""
|
|
1986
|
+
if not self.browser_profile.highlight_elements:
|
|
1987
|
+
return
|
|
1988
|
+
|
|
1989
|
+
try:
|
|
1990
|
+
# Get cached session
|
|
1991
|
+
cdp_session = await self.get_or_create_cdp_session()
|
|
1992
|
+
|
|
1993
|
+
# Remove highlights via JavaScript - be thorough
|
|
1994
|
+
script = """
|
|
1995
|
+
(function() {
|
|
1996
|
+
// Remove all browser-use highlight elements
|
|
1997
|
+
const highlights = document.querySelectorAll('[data-browser-use-highlight]');
|
|
1998
|
+
console.log('Removing', highlights.length, 'browser-use highlight elements');
|
|
1999
|
+
highlights.forEach(el => el.remove());
|
|
2000
|
+
|
|
2001
|
+
// Also remove by ID in case selector missed anything
|
|
2002
|
+
const highlightContainer = document.getElementById('browser-use-debug-highlights');
|
|
2003
|
+
if (highlightContainer) {
|
|
2004
|
+
console.log('Removing highlight container by ID');
|
|
2005
|
+
highlightContainer.remove();
|
|
2006
|
+
}
|
|
2007
|
+
|
|
2008
|
+
// Final cleanup - remove any orphaned tooltips
|
|
2009
|
+
const orphanedTooltips = document.querySelectorAll('[data-browser-use-highlight="tooltip"]');
|
|
2010
|
+
orphanedTooltips.forEach(el => el.remove());
|
|
2011
|
+
|
|
2012
|
+
return { removed: highlights.length };
|
|
2013
|
+
})();
|
|
2014
|
+
"""
|
|
2015
|
+
result = await cdp_session.cdp_client.send.Runtime.evaluate(
|
|
2016
|
+
params={'expression': script, 'returnByValue': True}, session_id=cdp_session.session_id
|
|
2017
|
+
)
|
|
2018
|
+
|
|
2019
|
+
# Log the result for debugging
|
|
2020
|
+
if result and 'result' in result and 'value' in result['result']:
|
|
2021
|
+
removed_count = result['result']['value'].get('removed', 0)
|
|
2022
|
+
self.logger.debug(f'Successfully removed {removed_count} highlight elements')
|
|
2023
|
+
else:
|
|
2024
|
+
self.logger.debug('Highlight removal completed')
|
|
2025
|
+
|
|
2026
|
+
except Exception as e:
|
|
2027
|
+
self.logger.warning(f'Failed to remove highlights: {e}')
|
|
2028
|
+
|
|
2029
|
+
@observe_debug(ignore_input=True, ignore_output=True, name='get_element_coordinates')
|
|
2030
|
+
async def get_element_coordinates(self, backend_node_id: int, cdp_session: CDPSession) -> DOMRect | None:
|
|
2031
|
+
"""Get element coordinates for a backend node ID using multiple methods.
|
|
2032
|
+
|
|
2033
|
+
This method tries DOM.getContentQuads first, then falls back to DOM.getBoxModel,
|
|
2034
|
+
and finally uses JavaScript getBoundingClientRect as a last resort.
|
|
2035
|
+
|
|
2036
|
+
Args:
|
|
2037
|
+
backend_node_id: The backend node ID to get coordinates for
|
|
2038
|
+
cdp_session: The CDP session to use
|
|
2039
|
+
|
|
2040
|
+
Returns:
|
|
2041
|
+
DOMRect with coordinates or None if element not found/no bounds
|
|
2042
|
+
"""
|
|
2043
|
+
session_id = cdp_session.session_id
|
|
2044
|
+
quads = []
|
|
2045
|
+
|
|
2046
|
+
# Method 1: Try DOM.getContentQuads first (best for inline elements and complex layouts)
|
|
2047
|
+
try:
|
|
2048
|
+
content_quads_result = await cdp_session.cdp_client.send.DOM.getContentQuads(
|
|
2049
|
+
params={'backendNodeId': backend_node_id}, session_id=session_id
|
|
2050
|
+
)
|
|
2051
|
+
if 'quads' in content_quads_result and content_quads_result['quads']:
|
|
2052
|
+
quads = content_quads_result['quads']
|
|
2053
|
+
self.logger.debug(f'Got {len(quads)} quads from DOM.getContentQuads')
|
|
2054
|
+
else:
|
|
2055
|
+
self.logger.debug(f'No quads found from DOM.getContentQuads {content_quads_result}')
|
|
2056
|
+
except Exception as e:
|
|
2057
|
+
self.logger.debug(f'DOM.getContentQuads failed: {e}')
|
|
2058
|
+
|
|
2059
|
+
# Method 2: Fall back to DOM.getBoxModel
|
|
2060
|
+
if not quads:
|
|
2061
|
+
try:
|
|
2062
|
+
box_model = await cdp_session.cdp_client.send.DOM.getBoxModel(
|
|
2063
|
+
params={'backendNodeId': backend_node_id}, session_id=session_id
|
|
2064
|
+
)
|
|
2065
|
+
if 'model' in box_model and 'content' in box_model['model']:
|
|
2066
|
+
content_quad = box_model['model']['content']
|
|
2067
|
+
if len(content_quad) >= 8:
|
|
2068
|
+
# Convert box model format to quad format
|
|
2069
|
+
quads = [
|
|
2070
|
+
[
|
|
2071
|
+
content_quad[0],
|
|
2072
|
+
content_quad[1], # x1, y1
|
|
2073
|
+
content_quad[2],
|
|
2074
|
+
content_quad[3], # x2, y2
|
|
2075
|
+
content_quad[4],
|
|
2076
|
+
content_quad[5], # x3, y3
|
|
2077
|
+
content_quad[6],
|
|
2078
|
+
content_quad[7], # x4, y4
|
|
2079
|
+
]
|
|
2080
|
+
]
|
|
2081
|
+
self.logger.debug('Got quad from DOM.getBoxModel')
|
|
2082
|
+
except Exception as e:
|
|
2083
|
+
self.logger.debug(f'DOM.getBoxModel failed: {e}')
|
|
2084
|
+
|
|
2085
|
+
# Method 3: Fall back to JavaScript getBoundingClientRect
|
|
2086
|
+
if not quads:
|
|
2087
|
+
try:
|
|
2088
|
+
result = await cdp_session.cdp_client.send.DOM.resolveNode(
|
|
2089
|
+
params={'backendNodeId': backend_node_id},
|
|
2090
|
+
session_id=session_id,
|
|
2091
|
+
)
|
|
2092
|
+
if 'object' in result and 'objectId' in result['object']:
|
|
2093
|
+
object_id = result['object']['objectId']
|
|
2094
|
+
js_result = await cdp_session.cdp_client.send.Runtime.callFunctionOn(
|
|
2095
|
+
params={
|
|
2096
|
+
'objectId': object_id,
|
|
2097
|
+
'functionDeclaration': """
|
|
2098
|
+
function() {
|
|
2099
|
+
const rect = this.getBoundingClientRect();
|
|
2100
|
+
return {
|
|
2101
|
+
x: rect.x,
|
|
2102
|
+
y: rect.y,
|
|
2103
|
+
width: rect.width,
|
|
2104
|
+
height: rect.height
|
|
2105
|
+
};
|
|
2106
|
+
}
|
|
2107
|
+
""",
|
|
2108
|
+
'returnByValue': True,
|
|
2109
|
+
},
|
|
2110
|
+
session_id=session_id,
|
|
2111
|
+
)
|
|
2112
|
+
if 'result' in js_result and 'value' in js_result['result']:
|
|
2113
|
+
rect_data = js_result['result']['value']
|
|
2114
|
+
if rect_data['width'] > 0 and rect_data['height'] > 0:
|
|
2115
|
+
return DOMRect(
|
|
2116
|
+
x=rect_data['x'], y=rect_data['y'], width=rect_data['width'], height=rect_data['height']
|
|
2117
|
+
)
|
|
2118
|
+
except Exception as e:
|
|
2119
|
+
self.logger.debug(f'JavaScript getBoundingClientRect failed: {e}')
|
|
2120
|
+
|
|
2121
|
+
# Convert quads to bounding rectangle if we have them
|
|
2122
|
+
if quads:
|
|
2123
|
+
# Use the first quad (most relevant for the element)
|
|
2124
|
+
quad = quads[0]
|
|
2125
|
+
if len(quad) >= 8:
|
|
2126
|
+
# Calculate bounding rect from quad points
|
|
2127
|
+
x_coords = [quad[i] for i in range(0, 8, 2)]
|
|
2128
|
+
y_coords = [quad[i] for i in range(1, 8, 2)]
|
|
2129
|
+
|
|
2130
|
+
min_x = min(x_coords)
|
|
2131
|
+
min_y = min(y_coords)
|
|
2132
|
+
max_x = max(x_coords)
|
|
2133
|
+
max_y = max(y_coords)
|
|
2134
|
+
|
|
2135
|
+
width = max_x - min_x
|
|
2136
|
+
height = max_y - min_y
|
|
2137
|
+
|
|
2138
|
+
if width > 0 and height > 0:
|
|
2139
|
+
return DOMRect(x=min_x, y=min_y, width=width, height=height)
|
|
2140
|
+
|
|
2141
|
+
return None
|
|
2142
|
+
|
|
2143
|
+
async def highlight_interaction_element(self, node: 'EnhancedDOMTreeNode') -> None:
|
|
2144
|
+
"""Temporarily highlight an element during interaction for user visibility.
|
|
2145
|
+
|
|
2146
|
+
This creates a visual highlight on the browser that shows the user which element
|
|
2147
|
+
is being interacted with. The highlight automatically fades after the configured duration.
|
|
2148
|
+
|
|
2149
|
+
Args:
|
|
2150
|
+
node: The DOM node to highlight with backend_node_id for coordinate lookup
|
|
2151
|
+
"""
|
|
2152
|
+
if not self.browser_profile.highlight_elements:
|
|
2153
|
+
return
|
|
2154
|
+
|
|
2155
|
+
try:
|
|
2156
|
+
import json
|
|
2157
|
+
|
|
2158
|
+
cdp_session = await self.get_or_create_cdp_session()
|
|
2159
|
+
|
|
2160
|
+
# Get current coordinates
|
|
2161
|
+
rect = await self.get_element_coordinates(node.backend_node_id, cdp_session)
|
|
2162
|
+
|
|
2163
|
+
color = self.browser_profile.interaction_highlight_color
|
|
2164
|
+
duration_ms = int(self.browser_profile.interaction_highlight_duration * 1000)
|
|
2165
|
+
|
|
2166
|
+
if not rect:
|
|
2167
|
+
self.logger.debug(f'No coordinates found for backend node {node.backend_node_id}')
|
|
2168
|
+
return
|
|
2169
|
+
|
|
2170
|
+
# Create animated corner brackets that start offset and animate inward
|
|
2171
|
+
script = f"""
|
|
2172
|
+
(function() {{
|
|
2173
|
+
const rect = {json.dumps({'x': rect.x, 'y': rect.y, 'width': rect.width, 'height': rect.height})};
|
|
2174
|
+
const color = {json.dumps(color)};
|
|
2175
|
+
const duration = {duration_ms};
|
|
2176
|
+
|
|
2177
|
+
// Scale corner size based on element dimensions to ensure gaps between corners
|
|
2178
|
+
const maxCornerSize = 20;
|
|
2179
|
+
const minCornerSize = 8;
|
|
2180
|
+
const cornerSize = Math.max(
|
|
2181
|
+
minCornerSize,
|
|
2182
|
+
Math.min(maxCornerSize, Math.min(rect.width, rect.height) * 0.35)
|
|
2183
|
+
);
|
|
2184
|
+
const borderWidth = 3;
|
|
2185
|
+
const startOffset = 10; // Starting offset in pixels
|
|
2186
|
+
const finalOffset = -3; // Final position slightly outside the element
|
|
2187
|
+
|
|
2188
|
+
// Get current scroll position
|
|
2189
|
+
const scrollX = window.pageXOffset || document.documentElement.scrollLeft || 0;
|
|
2190
|
+
const scrollY = window.pageYOffset || document.documentElement.scrollTop || 0;
|
|
2191
|
+
|
|
2192
|
+
// Create container for all corners
|
|
2193
|
+
const container = document.createElement('div');
|
|
2194
|
+
container.setAttribute('data-browser-use-interaction-highlight', 'true');
|
|
2195
|
+
container.style.cssText = `
|
|
2196
|
+
position: absolute;
|
|
2197
|
+
left: ${{rect.x + scrollX}}px;
|
|
2198
|
+
top: ${{rect.y + scrollY}}px;
|
|
2199
|
+
width: ${{rect.width}}px;
|
|
2200
|
+
height: ${{rect.height}}px;
|
|
2201
|
+
pointer-events: none;
|
|
2202
|
+
z-index: 2147483647;
|
|
2203
|
+
`;
|
|
2204
|
+
|
|
2205
|
+
// Create 4 corner brackets
|
|
2206
|
+
const corners = [
|
|
2207
|
+
{{ pos: 'top-left', startX: -startOffset, startY: -startOffset, finalX: finalOffset, finalY: finalOffset }},
|
|
2208
|
+
{{ pos: 'top-right', startX: startOffset, startY: -startOffset, finalX: -finalOffset, finalY: finalOffset }},
|
|
2209
|
+
{{ pos: 'bottom-left', startX: -startOffset, startY: startOffset, finalX: finalOffset, finalY: -finalOffset }},
|
|
2210
|
+
{{ pos: 'bottom-right', startX: startOffset, startY: startOffset, finalX: -finalOffset, finalY: -finalOffset }}
|
|
2211
|
+
];
|
|
2212
|
+
|
|
2213
|
+
corners.forEach(corner => {{
|
|
2214
|
+
const bracket = document.createElement('div');
|
|
2215
|
+
bracket.style.cssText = `
|
|
2216
|
+
position: absolute;
|
|
2217
|
+
width: ${{cornerSize}}px;
|
|
2218
|
+
height: ${{cornerSize}}px;
|
|
2219
|
+
pointer-events: none;
|
|
2220
|
+
transition: all 0.15s ease-out;
|
|
2221
|
+
`;
|
|
2222
|
+
|
|
2223
|
+
// Position corners
|
|
2224
|
+
if (corner.pos === 'top-left') {{
|
|
2225
|
+
bracket.style.top = '0';
|
|
2226
|
+
bracket.style.left = '0';
|
|
2227
|
+
bracket.style.borderTop = `${{borderWidth}}px solid ${{color}}`;
|
|
2228
|
+
bracket.style.borderLeft = `${{borderWidth}}px solid ${{color}}`;
|
|
2229
|
+
bracket.style.transform = `translate(${{corner.startX}}px, ${{corner.startY}}px)`;
|
|
2230
|
+
}} else if (corner.pos === 'top-right') {{
|
|
2231
|
+
bracket.style.top = '0';
|
|
2232
|
+
bracket.style.right = '0';
|
|
2233
|
+
bracket.style.borderTop = `${{borderWidth}}px solid ${{color}}`;
|
|
2234
|
+
bracket.style.borderRight = `${{borderWidth}}px solid ${{color}}`;
|
|
2235
|
+
bracket.style.transform = `translate(${{corner.startX}}px, ${{corner.startY}}px)`;
|
|
2236
|
+
}} else if (corner.pos === 'bottom-left') {{
|
|
2237
|
+
bracket.style.bottom = '0';
|
|
2238
|
+
bracket.style.left = '0';
|
|
2239
|
+
bracket.style.borderBottom = `${{borderWidth}}px solid ${{color}}`;
|
|
2240
|
+
bracket.style.borderLeft = `${{borderWidth}}px solid ${{color}}`;
|
|
2241
|
+
bracket.style.transform = `translate(${{corner.startX}}px, ${{corner.startY}}px)`;
|
|
2242
|
+
}} else if (corner.pos === 'bottom-right') {{
|
|
2243
|
+
bracket.style.bottom = '0';
|
|
2244
|
+
bracket.style.right = '0';
|
|
2245
|
+
bracket.style.borderBottom = `${{borderWidth}}px solid ${{color}}`;
|
|
2246
|
+
bracket.style.borderRight = `${{borderWidth}}px solid ${{color}}`;
|
|
2247
|
+
bracket.style.transform = `translate(${{corner.startX}}px, ${{corner.startY}}px)`;
|
|
2248
|
+
}}
|
|
2249
|
+
|
|
2250
|
+
container.appendChild(bracket);
|
|
2251
|
+
|
|
2252
|
+
// Animate to final position slightly outside the element
|
|
2253
|
+
setTimeout(() => {{
|
|
2254
|
+
bracket.style.transform = `translate(${{corner.finalX}}px, ${{corner.finalY}}px)`;
|
|
2255
|
+
}}, 10);
|
|
2256
|
+
}});
|
|
2257
|
+
|
|
2258
|
+
document.body.appendChild(container);
|
|
2259
|
+
|
|
2260
|
+
// Auto-remove after duration
|
|
2261
|
+
setTimeout(() => {{
|
|
2262
|
+
container.style.opacity = '0';
|
|
2263
|
+
container.style.transition = 'opacity 0.3s ease-out';
|
|
2264
|
+
setTimeout(() => container.remove(), 300);
|
|
2265
|
+
}}, duration);
|
|
2266
|
+
|
|
2267
|
+
return {{ created: true }};
|
|
2268
|
+
}})();
|
|
2269
|
+
"""
|
|
2270
|
+
|
|
2271
|
+
# Fire and forget - don't wait for completion
|
|
2272
|
+
|
|
2273
|
+
await cdp_session.cdp_client.send.Runtime.evaluate(
|
|
2274
|
+
params={'expression': script, 'returnByValue': True}, session_id=cdp_session.session_id
|
|
2275
|
+
)
|
|
2276
|
+
|
|
2277
|
+
except Exception as e:
|
|
2278
|
+
# Don't fail the action if highlighting fails
|
|
2279
|
+
self.logger.debug(f'Failed to highlight interaction element: {e}')
|
|
2280
|
+
|
|
2281
|
+
async def add_highlights(self, selector_map: dict[int, 'EnhancedDOMTreeNode']) -> None:
|
|
2282
|
+
"""Add visual highlights to the browser DOM for user visibility."""
|
|
2283
|
+
if not self.browser_profile.dom_highlight_elements or not selector_map:
|
|
2284
|
+
return
|
|
2285
|
+
|
|
2286
|
+
try:
|
|
2287
|
+
import json
|
|
2288
|
+
|
|
2289
|
+
# Convert selector_map to the format expected by the highlighting script
|
|
2290
|
+
elements_data = []
|
|
2291
|
+
for _, node in selector_map.items():
|
|
2292
|
+
# Get bounding box using absolute position (includes iframe translations) if available
|
|
2293
|
+
if node.absolute_position:
|
|
2294
|
+
# Use absolute position which includes iframe coordinate translations
|
|
2295
|
+
rect = node.absolute_position
|
|
2296
|
+
bbox = {'x': rect.x, 'y': rect.y, 'width': rect.width, 'height': rect.height}
|
|
2297
|
+
|
|
2298
|
+
# Only include elements with valid bounding boxes
|
|
2299
|
+
if bbox and bbox.get('width', 0) > 0 and bbox.get('height', 0) > 0:
|
|
2300
|
+
element = {
|
|
2301
|
+
'x': bbox['x'],
|
|
2302
|
+
'y': bbox['y'],
|
|
2303
|
+
'width': bbox['width'],
|
|
2304
|
+
'height': bbox['height'],
|
|
2305
|
+
'element_name': node.node_name,
|
|
2306
|
+
'is_clickable': node.snapshot_node.is_clickable if node.snapshot_node else True,
|
|
2307
|
+
'is_scrollable': getattr(node, 'is_scrollable', False),
|
|
2308
|
+
'attributes': node.attributes or {},
|
|
2309
|
+
'frame_id': getattr(node, 'frame_id', None),
|
|
2310
|
+
'node_id': node.node_id,
|
|
2311
|
+
'backend_node_id': node.backend_node_id,
|
|
2312
|
+
'xpath': node.xpath,
|
|
2313
|
+
'text_content': node.get_all_children_text()[:50]
|
|
2314
|
+
if hasattr(node, 'get_all_children_text')
|
|
2315
|
+
else node.node_value[:50],
|
|
2316
|
+
}
|
|
2317
|
+
elements_data.append(element)
|
|
2318
|
+
|
|
2319
|
+
if not elements_data:
|
|
2320
|
+
self.logger.debug('⚠️ No valid elements to highlight')
|
|
2321
|
+
return
|
|
2322
|
+
|
|
2323
|
+
self.logger.debug(f'📍 Creating highlights for {len(elements_data)} elements')
|
|
2324
|
+
|
|
2325
|
+
# Always remove existing highlights first
|
|
2326
|
+
await self.remove_highlights()
|
|
2327
|
+
|
|
2328
|
+
# Add a small delay to ensure removal completes
|
|
2329
|
+
import asyncio
|
|
2330
|
+
|
|
2331
|
+
await asyncio.sleep(0.05)
|
|
2332
|
+
|
|
2333
|
+
# Get CDP session
|
|
2334
|
+
cdp_session = await self.get_or_create_cdp_session()
|
|
2335
|
+
|
|
2336
|
+
# Create the proven highlighting script from v0.6.0 with fixed positioning
|
|
2337
|
+
script = f"""
|
|
2338
|
+
(function() {{
|
|
2339
|
+
// Interactive elements data
|
|
2340
|
+
const interactiveElements = {json.dumps(elements_data)};
|
|
2341
|
+
|
|
2342
|
+
console.log('=== BROWSER-USE HIGHLIGHTING ===');
|
|
2343
|
+
console.log('Highlighting', interactiveElements.length, 'interactive elements');
|
|
2344
|
+
|
|
2345
|
+
// Double-check: Remove any existing highlight container first
|
|
2346
|
+
const existingContainer = document.getElementById('browser-use-debug-highlights');
|
|
2347
|
+
if (existingContainer) {{
|
|
2348
|
+
console.log('⚠️ Found existing highlight container, removing it first');
|
|
2349
|
+
existingContainer.remove();
|
|
2350
|
+
}}
|
|
2351
|
+
|
|
2352
|
+
// Also remove any stray highlight elements
|
|
2353
|
+
const strayHighlights = document.querySelectorAll('[data-browser-use-highlight]');
|
|
2354
|
+
if (strayHighlights.length > 0) {{
|
|
2355
|
+
console.log('⚠️ Found', strayHighlights.length, 'stray highlight elements, removing them');
|
|
2356
|
+
strayHighlights.forEach(el => el.remove());
|
|
2357
|
+
}}
|
|
2358
|
+
|
|
2359
|
+
// Use maximum z-index for visibility
|
|
2360
|
+
const HIGHLIGHT_Z_INDEX = 2147483647;
|
|
2361
|
+
|
|
2362
|
+
// Create container for all highlights - use FIXED positioning (key insight from v0.6.0)
|
|
2363
|
+
const container = document.createElement('div');
|
|
2364
|
+
container.id = 'browser-use-debug-highlights';
|
|
2365
|
+
container.setAttribute('data-browser-use-highlight', 'container');
|
|
2366
|
+
|
|
2367
|
+
container.style.cssText = `
|
|
2368
|
+
position: absolute;
|
|
2369
|
+
top: 0;
|
|
2370
|
+
left: 0;
|
|
2371
|
+
width: 100vw;
|
|
2372
|
+
height: 100vh;
|
|
2373
|
+
pointer-events: none;
|
|
2374
|
+
z-index: ${{HIGHLIGHT_Z_INDEX}};
|
|
2375
|
+
overflow: visible;
|
|
2376
|
+
margin: 0;
|
|
2377
|
+
padding: 0;
|
|
2378
|
+
border: none;
|
|
2379
|
+
outline: none;
|
|
2380
|
+
box-shadow: none;
|
|
2381
|
+
background: none;
|
|
2382
|
+
font-family: inherit;
|
|
2383
|
+
`;
|
|
2384
|
+
|
|
2385
|
+
// Helper function to create text elements safely
|
|
2386
|
+
function createTextElement(tag, text, styles) {{
|
|
2387
|
+
const element = document.createElement(tag);
|
|
2388
|
+
element.textContent = text;
|
|
2389
|
+
if (styles) element.style.cssText = styles;
|
|
2390
|
+
return element;
|
|
2391
|
+
}}
|
|
2392
|
+
|
|
2393
|
+
// Add highlights for each element
|
|
2394
|
+
interactiveElements.forEach((element, index) => {{
|
|
2395
|
+
const highlight = document.createElement('div');
|
|
2396
|
+
highlight.setAttribute('data-browser-use-highlight', 'element');
|
|
2397
|
+
highlight.setAttribute('data-element-id', element.backend_node_id);
|
|
2398
|
+
highlight.style.cssText = `
|
|
2399
|
+
position: absolute;
|
|
2400
|
+
left: ${{element.x}}px;
|
|
2401
|
+
top: ${{element.y}}px;
|
|
2402
|
+
width: ${{element.width}}px;
|
|
2403
|
+
height: ${{element.height}}px;
|
|
2404
|
+
outline: 2px dashed #4a90e2;
|
|
2405
|
+
outline-offset: -2px;
|
|
2406
|
+
background: transparent;
|
|
2407
|
+
pointer-events: none;
|
|
2408
|
+
box-sizing: content-box;
|
|
2409
|
+
transition: outline 0.2s ease;
|
|
2410
|
+
margin: 0;
|
|
2411
|
+
padding: 0;
|
|
2412
|
+
border: none;
|
|
2413
|
+
`;
|
|
2414
|
+
|
|
2415
|
+
// Enhanced label with backend node ID
|
|
2416
|
+
const label = createTextElement('div', element.backend_node_id, `
|
|
2417
|
+
position: absolute;
|
|
2418
|
+
top: -20px;
|
|
2419
|
+
left: 0;
|
|
2420
|
+
background-color: #4a90e2;
|
|
2421
|
+
color: white;
|
|
2422
|
+
padding: 2px 6px;
|
|
2423
|
+
font-size: 11px;
|
|
2424
|
+
font-family: 'Monaco', 'Menlo', 'Ubuntu Mono', monospace;
|
|
2425
|
+
font-weight: bold;
|
|
2426
|
+
border-radius: 3px;
|
|
2427
|
+
white-space: nowrap;
|
|
2428
|
+
z-index: ${{HIGHLIGHT_Z_INDEX + 1}};
|
|
2429
|
+
box-shadow: 0 2px 4px rgba(0,0,0,0.3);
|
|
2430
|
+
border: none;
|
|
2431
|
+
outline: none;
|
|
2432
|
+
margin: 0;
|
|
2433
|
+
line-height: 1.2;
|
|
2434
|
+
`);
|
|
2435
|
+
|
|
2436
|
+
highlight.appendChild(label);
|
|
2437
|
+
container.appendChild(highlight);
|
|
2438
|
+
}});
|
|
2439
|
+
|
|
2440
|
+
// Add container to document
|
|
2441
|
+
document.body.appendChild(container);
|
|
2442
|
+
|
|
2443
|
+
console.log('Highlighting complete - added', interactiveElements.length, 'highlights');
|
|
2444
|
+
return {{ added: interactiveElements.length }};
|
|
2445
|
+
}})();
|
|
2446
|
+
"""
|
|
2447
|
+
|
|
2448
|
+
# Execute the script
|
|
2449
|
+
result = await cdp_session.cdp_client.send.Runtime.evaluate(
|
|
2450
|
+
params={'expression': script, 'returnByValue': True}, session_id=cdp_session.session_id
|
|
2451
|
+
)
|
|
2452
|
+
|
|
2453
|
+
# Log the result
|
|
2454
|
+
if result and 'result' in result and 'value' in result['result']:
|
|
2455
|
+
added_count = result['result']['value'].get('added', 0)
|
|
2456
|
+
self.logger.debug(f'Successfully added {added_count} highlight elements to browser DOM')
|
|
2457
|
+
else:
|
|
2458
|
+
self.logger.debug('Browser highlight injection completed')
|
|
2459
|
+
|
|
2460
|
+
except Exception as e:
|
|
2461
|
+
self.logger.warning(f'Failed to add browser highlights: {e}')
|
|
2462
|
+
import traceback
|
|
2463
|
+
|
|
2464
|
+
self.logger.debug(f'Browser highlight traceback: {traceback.format_exc()}')
|
|
2465
|
+
|
|
2466
|
+
async def _close_extension_options_pages(self) -> None:
|
|
2467
|
+
"""Close any extension options/welcome pages that have opened."""
|
|
2468
|
+
try:
|
|
2469
|
+
# Get all open pages
|
|
2470
|
+
targets = await self._cdp_get_all_pages()
|
|
2471
|
+
|
|
2472
|
+
for target in targets:
|
|
2473
|
+
target_url = target.get('url', '')
|
|
2474
|
+
target_id = target.get('targetId', '')
|
|
2475
|
+
|
|
2476
|
+
# Check if this is an extension options/welcome page
|
|
2477
|
+
if 'chrome-extension://' in target_url and (
|
|
2478
|
+
'options.html' in target_url or 'welcome.html' in target_url or 'onboarding.html' in target_url
|
|
2479
|
+
):
|
|
2480
|
+
self.logger.info(f'[BrowserSession] 🚫 Closing extension options page: {target_url}')
|
|
2481
|
+
try:
|
|
2482
|
+
await self._cdp_close_page(target_id)
|
|
2483
|
+
except Exception as e:
|
|
2484
|
+
self.logger.debug(f'[BrowserSession] Could not close extension page {target_id}: {e}')
|
|
2485
|
+
|
|
2486
|
+
except Exception as e:
|
|
2487
|
+
self.logger.debug(f'[BrowserSession] Error closing extension options pages: {e}')
|
|
2488
|
+
|
|
2489
|
+
@property
|
|
2490
|
+
def downloaded_files(self) -> list[str]:
|
|
2491
|
+
"""Get list of files downloaded during this browser session.
|
|
2492
|
+
|
|
2493
|
+
Returns:
|
|
2494
|
+
list[str]: List of absolute file paths to downloaded files in this session
|
|
2495
|
+
"""
|
|
2496
|
+
return self._downloaded_files.copy()
|
|
2497
|
+
|
|
2498
|
+
# endregion - ========== Helper Methods ==========
|
|
2499
|
+
|
|
2500
|
+
# region - ========== CDP-based replacements for browser_context operations ==========
|
|
2501
|
+
|
|
2502
|
+
async def _cdp_get_all_pages(
|
|
2503
|
+
self,
|
|
2504
|
+
include_http: bool = True,
|
|
2505
|
+
include_about: bool = True,
|
|
2506
|
+
include_pages: bool = True,
|
|
2507
|
+
include_iframes: bool = False,
|
|
2508
|
+
include_workers: bool = False,
|
|
2509
|
+
include_chrome: bool = False,
|
|
2510
|
+
include_chrome_extensions: bool = False,
|
|
2511
|
+
include_chrome_error: bool = False,
|
|
2512
|
+
) -> list[TargetInfo]:
|
|
2513
|
+
"""Get all browser pages/tabs using CDP Target.getTargets."""
|
|
2514
|
+
# Safety check - return empty list if browser not connected yet
|
|
2515
|
+
if not self._cdp_client_root:
|
|
2516
|
+
return []
|
|
2517
|
+
targets = await self.cdp_client.send.Target.getTargets()
|
|
2518
|
+
# Filter for valid page/tab targets only
|
|
2519
|
+
return [
|
|
2520
|
+
t
|
|
2521
|
+
for t in targets.get('targetInfos', [])
|
|
2522
|
+
if self._is_valid_target(
|
|
2523
|
+
t,
|
|
2524
|
+
include_http=include_http,
|
|
2525
|
+
include_about=include_about,
|
|
2526
|
+
include_pages=include_pages,
|
|
2527
|
+
include_iframes=include_iframes,
|
|
2528
|
+
include_workers=include_workers,
|
|
2529
|
+
include_chrome=include_chrome,
|
|
2530
|
+
include_chrome_extensions=include_chrome_extensions,
|
|
2531
|
+
include_chrome_error=include_chrome_error,
|
|
2532
|
+
)
|
|
2533
|
+
]
|
|
2534
|
+
|
|
2535
|
+
async def _cdp_create_new_page(self, url: str = 'about:blank', background: bool = False, new_window: bool = False) -> str:
|
|
2536
|
+
"""Create a new page/tab using CDP Target.createTarget. Returns target ID."""
|
|
2537
|
+
# Use the root CDP client to create tabs at the browser level
|
|
2538
|
+
if self._cdp_client_root:
|
|
2539
|
+
result = await self._cdp_client_root.send.Target.createTarget(
|
|
2540
|
+
params={'url': url, 'newWindow': new_window, 'background': background}
|
|
2541
|
+
)
|
|
2542
|
+
else:
|
|
2543
|
+
# Fallback to using cdp_client if root is not available
|
|
2544
|
+
result = await self.cdp_client.send.Target.createTarget(
|
|
2545
|
+
params={'url': url, 'newWindow': new_window, 'background': background}
|
|
2546
|
+
)
|
|
2547
|
+
return result['targetId']
|
|
2548
|
+
|
|
2549
|
+
async def _cdp_close_page(self, target_id: TargetID) -> None:
|
|
2550
|
+
"""Close a page/tab using CDP Target.closeTarget."""
|
|
2551
|
+
await self.cdp_client.send.Target.closeTarget(params={'targetId': target_id})
|
|
2552
|
+
|
|
2553
|
+
async def _cdp_get_cookies(self) -> list[Cookie]:
|
|
2554
|
+
"""Get cookies using CDP Network.getCookies."""
|
|
2555
|
+
cdp_session = await self.get_or_create_cdp_session(target_id=None)
|
|
2556
|
+
result = await asyncio.wait_for(
|
|
2557
|
+
cdp_session.cdp_client.send.Storage.getCookies(session_id=cdp_session.session_id), timeout=8.0
|
|
2558
|
+
)
|
|
2559
|
+
return result.get('cookies', [])
|
|
2560
|
+
|
|
2561
|
+
async def _cdp_set_cookies(self, cookies: list[Cookie]) -> None:
|
|
2562
|
+
"""Set cookies using CDP Storage.setCookies."""
|
|
2563
|
+
if not self.agent_focus or not cookies:
|
|
2564
|
+
return
|
|
2565
|
+
|
|
2566
|
+
cdp_session = await self.get_or_create_cdp_session(target_id=None)
|
|
2567
|
+
# Storage.setCookies expects params dict with 'cookies' key
|
|
2568
|
+
await cdp_session.cdp_client.send.Storage.setCookies(
|
|
2569
|
+
params={'cookies': cookies}, # type: ignore[arg-type]
|
|
2570
|
+
session_id=cdp_session.session_id,
|
|
2571
|
+
)
|
|
2572
|
+
|
|
2573
|
+
async def _cdp_clear_cookies(self) -> None:
|
|
2574
|
+
"""Clear all cookies using CDP Network.clearBrowserCookies."""
|
|
2575
|
+
cdp_session = await self.get_or_create_cdp_session()
|
|
2576
|
+
await cdp_session.cdp_client.send.Storage.clearCookies(session_id=cdp_session.session_id)
|
|
2577
|
+
|
|
2578
|
+
async def _cdp_set_extra_headers(self, headers: dict[str, str]) -> None:
|
|
2579
|
+
"""Set extra HTTP headers using CDP Network.setExtraHTTPHeaders."""
|
|
2580
|
+
if not self.agent_focus:
|
|
2581
|
+
return
|
|
2582
|
+
|
|
2583
|
+
cdp_session = await self.get_or_create_cdp_session()
|
|
2584
|
+
# await cdp_session.cdp_client.send.Network.setExtraHTTPHeaders(params={'headers': headers}, session_id=cdp_session.session_id)
|
|
2585
|
+
raise NotImplementedError('Not implemented yet')
|
|
2586
|
+
|
|
2587
|
+
async def _cdp_grant_permissions(self, permissions: list[str], origin: str | None = None) -> None:
|
|
2588
|
+
"""Grant permissions using CDP Browser.grantPermissions."""
|
|
2589
|
+
params = {'permissions': permissions}
|
|
2590
|
+
# if origin:
|
|
2591
|
+
# params['origin'] = origin
|
|
2592
|
+
cdp_session = await self.get_or_create_cdp_session()
|
|
2593
|
+
# await cdp_session.cdp_client.send.Browser.grantPermissions(params=params, session_id=cdp_session.session_id)
|
|
2594
|
+
raise NotImplementedError('Not implemented yet')
|
|
2595
|
+
|
|
2596
|
+
async def _cdp_set_geolocation(self, latitude: float, longitude: float, accuracy: float = 100) -> None:
|
|
2597
|
+
"""Set geolocation using CDP Emulation.setGeolocationOverride."""
|
|
2598
|
+
await self.cdp_client.send.Emulation.setGeolocationOverride(
|
|
2599
|
+
params={'latitude': latitude, 'longitude': longitude, 'accuracy': accuracy}
|
|
2600
|
+
)
|
|
2601
|
+
|
|
2602
|
+
async def _cdp_clear_geolocation(self) -> None:
|
|
2603
|
+
"""Clear geolocation override using CDP."""
|
|
2604
|
+
await self.cdp_client.send.Emulation.clearGeolocationOverride()
|
|
2605
|
+
|
|
2606
|
+
async def _cdp_add_init_script(self, script: str) -> str:
|
|
2607
|
+
"""Add script to evaluate on new document using CDP Page.addScriptToEvaluateOnNewDocument."""
|
|
2608
|
+
assert self._cdp_client_root is not None
|
|
2609
|
+
cdp_session = await self.get_or_create_cdp_session()
|
|
2610
|
+
|
|
2611
|
+
result = await cdp_session.cdp_client.send.Page.addScriptToEvaluateOnNewDocument(
|
|
2612
|
+
params={'source': script, 'runImmediately': True}, session_id=cdp_session.session_id
|
|
2613
|
+
)
|
|
2614
|
+
return result['identifier']
|
|
2615
|
+
|
|
2616
|
+
async def _cdp_remove_init_script(self, identifier: str) -> None:
|
|
2617
|
+
"""Remove script added with addScriptToEvaluateOnNewDocument."""
|
|
2618
|
+
cdp_session = await self.get_or_create_cdp_session(target_id=None)
|
|
2619
|
+
await cdp_session.cdp_client.send.Page.removeScriptToEvaluateOnNewDocument(
|
|
2620
|
+
params={'identifier': identifier}, session_id=cdp_session.session_id
|
|
2621
|
+
)
|
|
2622
|
+
|
|
2623
|
+
async def _cdp_set_viewport(
|
|
2624
|
+
self, width: int, height: int, device_scale_factor: float = 1.0, mobile: bool = False, target_id: str | None = None
|
|
2625
|
+
) -> None:
|
|
2626
|
+
"""Set viewport using CDP Emulation.setDeviceMetricsOverride.
|
|
2627
|
+
|
|
2628
|
+
Args:
|
|
2629
|
+
width: Viewport width
|
|
2630
|
+
height: Viewport height
|
|
2631
|
+
device_scale_factor: Device scale factor (default 1.0)
|
|
2632
|
+
mobile: Whether to emulate mobile device (default False)
|
|
2633
|
+
target_id: Optional target ID to set viewport for. If not provided, uses agent_focus.
|
|
2634
|
+
"""
|
|
2635
|
+
if target_id:
|
|
2636
|
+
# Set viewport for specific target
|
|
2637
|
+
cdp_session = await self.get_or_create_cdp_session(target_id, focus=False)
|
|
2638
|
+
elif self.agent_focus:
|
|
2639
|
+
# Use current focus
|
|
2640
|
+
cdp_session = self.agent_focus
|
|
2641
|
+
else:
|
|
2642
|
+
self.logger.warning('Cannot set viewport: no target_id provided and agent_focus not initialized')
|
|
2643
|
+
return
|
|
2644
|
+
|
|
2645
|
+
await cdp_session.cdp_client.send.Emulation.setDeviceMetricsOverride(
|
|
2646
|
+
params={'width': width, 'height': height, 'deviceScaleFactor': device_scale_factor, 'mobile': mobile},
|
|
2647
|
+
session_id=cdp_session.session_id,
|
|
2648
|
+
)
|
|
2649
|
+
|
|
2650
|
+
async def _cdp_get_origins(self) -> list[dict[str, Any]]:
|
|
2651
|
+
"""Get origins with localStorage and sessionStorage using CDP."""
|
|
2652
|
+
origins = []
|
|
2653
|
+
cdp_session = await self.get_or_create_cdp_session(target_id=None)
|
|
2654
|
+
|
|
2655
|
+
try:
|
|
2656
|
+
# Enable DOMStorage domain to track storage
|
|
2657
|
+
await cdp_session.cdp_client.send.DOMStorage.enable(session_id=cdp_session.session_id)
|
|
2658
|
+
|
|
2659
|
+
try:
|
|
2660
|
+
# Get all frames to find unique origins
|
|
2661
|
+
frames_result = await cdp_session.cdp_client.send.Page.getFrameTree(session_id=cdp_session.session_id)
|
|
2662
|
+
|
|
2663
|
+
# Extract unique origins from frames
|
|
2664
|
+
unique_origins = set()
|
|
2665
|
+
|
|
2666
|
+
def _extract_origins(frame_tree):
|
|
2667
|
+
"""Recursively extract origins from frame tree."""
|
|
2668
|
+
frame = frame_tree.get('frame', {})
|
|
2669
|
+
origin = frame.get('securityOrigin')
|
|
2670
|
+
if origin and origin != 'null':
|
|
2671
|
+
unique_origins.add(origin)
|
|
2672
|
+
|
|
2673
|
+
# Process child frames
|
|
2674
|
+
for child in frame_tree.get('childFrames', []):
|
|
2675
|
+
_extract_origins(child)
|
|
2676
|
+
|
|
2677
|
+
async def _get_storage_items(origin: str, is_local_storage: bool) -> list[dict[str, str]] | None:
|
|
2678
|
+
"""Helper to get storage items for an origin."""
|
|
2679
|
+
storage_type = 'localStorage' if is_local_storage else 'sessionStorage'
|
|
2680
|
+
try:
|
|
2681
|
+
result = await cdp_session.cdp_client.send.DOMStorage.getDOMStorageItems(
|
|
2682
|
+
params={'storageId': {'securityOrigin': origin, 'isLocalStorage': is_local_storage}},
|
|
2683
|
+
session_id=cdp_session.session_id,
|
|
2684
|
+
)
|
|
2685
|
+
|
|
2686
|
+
items = []
|
|
2687
|
+
for item in result.get('entries', []):
|
|
2688
|
+
if len(item) == 2: # Each item is [key, value]
|
|
2689
|
+
items.append({'name': item[0], 'value': item[1]})
|
|
2690
|
+
|
|
2691
|
+
return items if items else None
|
|
2692
|
+
except Exception as e:
|
|
2693
|
+
self.logger.debug(f'Failed to get {storage_type} for {origin}: {e}')
|
|
2694
|
+
return None
|
|
2695
|
+
|
|
2696
|
+
_extract_origins(frames_result.get('frameTree', {}))
|
|
2697
|
+
|
|
2698
|
+
# For each unique origin, get localStorage and sessionStorage
|
|
2699
|
+
for origin in unique_origins:
|
|
2700
|
+
origin_data = {'origin': origin}
|
|
2701
|
+
|
|
2702
|
+
# Get localStorage
|
|
2703
|
+
local_storage = await _get_storage_items(origin, is_local_storage=True)
|
|
2704
|
+
if local_storage:
|
|
2705
|
+
origin_data['localStorage'] = local_storage
|
|
2706
|
+
|
|
2707
|
+
# Get sessionStorage
|
|
2708
|
+
session_storage = await _get_storage_items(origin, is_local_storage=False)
|
|
2709
|
+
if session_storage:
|
|
2710
|
+
origin_data['sessionStorage'] = session_storage
|
|
2711
|
+
|
|
2712
|
+
# Only add origin if it has storage data
|
|
2713
|
+
if 'localStorage' in origin_data or 'sessionStorage' in origin_data:
|
|
2714
|
+
origins.append(origin_data)
|
|
2715
|
+
|
|
2716
|
+
finally:
|
|
2717
|
+
# Always disable DOMStorage tracking when done
|
|
2718
|
+
await cdp_session.cdp_client.send.DOMStorage.disable(session_id=cdp_session.session_id)
|
|
2719
|
+
|
|
2720
|
+
except Exception as e:
|
|
2721
|
+
self.logger.warning(f'Failed to get origins: {e}')
|
|
2722
|
+
|
|
2723
|
+
return origins
|
|
2724
|
+
|
|
2725
|
+
async def _cdp_get_storage_state(self) -> dict:
|
|
2726
|
+
"""Get storage state (cookies, localStorage, sessionStorage) using CDP."""
|
|
2727
|
+
# Use the _cdp_get_cookies helper which handles session attachment
|
|
2728
|
+
cookies = await self._cdp_get_cookies()
|
|
2729
|
+
|
|
2730
|
+
# Get origins with localStorage/sessionStorage
|
|
2731
|
+
origins = await self._cdp_get_origins()
|
|
2732
|
+
|
|
2733
|
+
return {
|
|
2734
|
+
'cookies': cookies,
|
|
2735
|
+
'origins': origins,
|
|
2736
|
+
}
|
|
2737
|
+
|
|
2738
|
+
async def _cdp_navigate(self, url: str, target_id: TargetID | None = None) -> None:
|
|
2739
|
+
"""Navigate to URL using CDP Page.navigate."""
|
|
2740
|
+
# Use provided target_id or fall back to current_target_id
|
|
2741
|
+
|
|
2742
|
+
assert self._cdp_client_root is not None, 'CDP client not initialized - browser may not be connected yet'
|
|
2743
|
+
assert self.agent_focus is not None, 'CDP session not initialized - browser may not be connected yet'
|
|
2744
|
+
|
|
2745
|
+
self.agent_focus = await self.get_or_create_cdp_session(target_id or self.agent_focus.target_id, focus=True)
|
|
2746
|
+
|
|
2747
|
+
# Use helper to navigate on the target
|
|
2748
|
+
await self.agent_focus.cdp_client.send.Page.navigate(params={'url': url}, session_id=self.agent_focus.session_id)
|
|
2749
|
+
|
|
2750
|
+
@staticmethod
|
|
2751
|
+
def _is_valid_target(
|
|
2752
|
+
target_info: TargetInfo,
|
|
2753
|
+
include_http: bool = True,
|
|
2754
|
+
include_chrome: bool = False,
|
|
2755
|
+
include_chrome_extensions: bool = False,
|
|
2756
|
+
include_chrome_error: bool = False,
|
|
2757
|
+
include_about: bool = True,
|
|
2758
|
+
include_iframes: bool = True,
|
|
2759
|
+
include_pages: bool = True,
|
|
2760
|
+
include_workers: bool = False,
|
|
2761
|
+
) -> bool:
|
|
2762
|
+
"""Check if a target should be processed.
|
|
2763
|
+
|
|
2764
|
+
Args:
|
|
2765
|
+
target_info: Target info dict from CDP
|
|
2766
|
+
|
|
2767
|
+
Returns:
|
|
2768
|
+
True if target should be processed, False if it should be skipped
|
|
2769
|
+
"""
|
|
2770
|
+
target_type = target_info.get('type', '')
|
|
2771
|
+
url = target_info.get('url', '')
|
|
2772
|
+
|
|
2773
|
+
url_allowed, type_allowed = False, False
|
|
2774
|
+
|
|
2775
|
+
# Always allow new tab pages (chrome://new-tab-page/, chrome://newtab/, about:blank)
|
|
2776
|
+
# so they can be redirected to about:blank in connect()
|
|
2777
|
+
from browser_use.utils import is_new_tab_page
|
|
2778
|
+
|
|
2779
|
+
if is_new_tab_page(url):
|
|
2780
|
+
url_allowed = True
|
|
2781
|
+
|
|
2782
|
+
if url.startswith('chrome-error://') and include_chrome_error:
|
|
2783
|
+
url_allowed = True
|
|
2784
|
+
|
|
2785
|
+
if url.startswith('chrome://') and include_chrome:
|
|
2786
|
+
url_allowed = True
|
|
2787
|
+
|
|
2788
|
+
if url.startswith('chrome-extension://') and include_chrome_extensions:
|
|
2789
|
+
url_allowed = True
|
|
2790
|
+
|
|
2791
|
+
# dont allow about:srcdoc! there are also other rare about: pages that we want to avoid
|
|
2792
|
+
if url == 'about:blank' and include_about:
|
|
2793
|
+
url_allowed = True
|
|
2794
|
+
|
|
2795
|
+
if (url.startswith('http://') or url.startswith('https://')) and include_http:
|
|
2796
|
+
url_allowed = True
|
|
2797
|
+
|
|
2798
|
+
if target_type in ('service_worker', 'shared_worker', 'worker') and include_workers:
|
|
2799
|
+
type_allowed = True
|
|
2800
|
+
|
|
2801
|
+
if target_type in ('page', 'tab') and include_pages:
|
|
2802
|
+
type_allowed = True
|
|
2803
|
+
|
|
2804
|
+
if target_type in ('iframe', 'webview') and include_iframes:
|
|
2805
|
+
type_allowed = True
|
|
2806
|
+
|
|
2807
|
+
return url_allowed and type_allowed
|
|
2808
|
+
|
|
2809
|
+
async def get_all_frames(self) -> tuple[dict[str, dict], dict[str, str]]:
|
|
2810
|
+
"""Get a complete frame hierarchy from all browser targets.
|
|
2811
|
+
|
|
2812
|
+
Returns:
|
|
2813
|
+
Tuple of (all_frames, target_sessions) where:
|
|
2814
|
+
- all_frames: dict mapping frame_id -> frame info dict with all metadata
|
|
2815
|
+
- target_sessions: dict mapping target_id -> session_id for active sessions
|
|
2816
|
+
"""
|
|
2817
|
+
all_frames = {} # frame_id -> FrameInfo dict
|
|
2818
|
+
target_sessions = {} # target_id -> session_id (keep sessions alive during collection)
|
|
2819
|
+
|
|
2820
|
+
# Check if cross-origin iframe support is enabled
|
|
2821
|
+
include_cross_origin = self.browser_profile.cross_origin_iframes
|
|
2822
|
+
|
|
2823
|
+
# Get all targets - only include iframes if cross-origin support is enabled
|
|
2824
|
+
targets = await self._cdp_get_all_pages(
|
|
2825
|
+
include_http=True,
|
|
2826
|
+
include_about=True,
|
|
2827
|
+
include_pages=True,
|
|
2828
|
+
include_iframes=include_cross_origin, # Only include iframe targets if flag is set
|
|
2829
|
+
include_workers=False,
|
|
2830
|
+
include_chrome=False,
|
|
2831
|
+
include_chrome_extensions=False,
|
|
2832
|
+
include_chrome_error=include_cross_origin, # Only include error pages if cross-origin is enabled
|
|
2833
|
+
)
|
|
2834
|
+
all_targets = targets
|
|
2835
|
+
|
|
2836
|
+
# First pass: collect frame trees from ALL targets
|
|
2837
|
+
for target in all_targets:
|
|
2838
|
+
target_id = target['targetId']
|
|
2839
|
+
|
|
2840
|
+
# Skip iframe targets if cross-origin support is disabled
|
|
2841
|
+
if not include_cross_origin and target.get('type') == 'iframe':
|
|
2842
|
+
continue
|
|
2843
|
+
|
|
2844
|
+
# When cross-origin support is disabled, only process the current target
|
|
2845
|
+
if not include_cross_origin:
|
|
2846
|
+
# Only process the current focus target
|
|
2847
|
+
if self.agent_focus and target_id != self.agent_focus.target_id:
|
|
2848
|
+
continue
|
|
2849
|
+
# Use the existing agent_focus session
|
|
2850
|
+
cdp_session = self.agent_focus
|
|
2851
|
+
else:
|
|
2852
|
+
# Get cached session for this target (don't change focus - iterating frames)
|
|
2853
|
+
cdp_session = await self.get_or_create_cdp_session(target_id, focus=False)
|
|
2854
|
+
|
|
2855
|
+
if cdp_session:
|
|
2856
|
+
target_sessions[target_id] = cdp_session.session_id
|
|
2857
|
+
|
|
2858
|
+
try:
|
|
2859
|
+
# Try to get frame tree (not all target types support this)
|
|
2860
|
+
frame_tree_result = await cdp_session.cdp_client.send.Page.getFrameTree(session_id=cdp_session.session_id)
|
|
2861
|
+
|
|
2862
|
+
# Process the frame tree recursively
|
|
2863
|
+
def process_frame_tree(node, parent_frame_id=None):
|
|
2864
|
+
"""Recursively process frame tree and add to all_frames."""
|
|
2865
|
+
frame = node.get('frame', {})
|
|
2866
|
+
current_frame_id = frame.get('id')
|
|
2867
|
+
|
|
2868
|
+
if current_frame_id:
|
|
2869
|
+
# For iframe targets, check if the frame has a parentId field
|
|
2870
|
+
# This indicates it's an OOPIF with a parent in another target
|
|
2871
|
+
actual_parent_id = frame.get('parentId') or parent_frame_id
|
|
2872
|
+
|
|
2873
|
+
# Create frame info with all CDP response data plus our additions
|
|
2874
|
+
frame_info = {
|
|
2875
|
+
**frame, # Include all original frame data: id, url, parentId, etc.
|
|
2876
|
+
'frameTargetId': target_id, # Target that can access this frame
|
|
2877
|
+
'parentFrameId': actual_parent_id, # Use parentId from frame if available
|
|
2878
|
+
'childFrameIds': [], # Will be populated below
|
|
2879
|
+
'isCrossOrigin': False, # Will be determined based on context
|
|
2880
|
+
'isValidTarget': self._is_valid_target(
|
|
2881
|
+
target,
|
|
2882
|
+
include_http=True,
|
|
2883
|
+
include_about=True,
|
|
2884
|
+
include_pages=True,
|
|
2885
|
+
include_iframes=True,
|
|
2886
|
+
include_workers=False,
|
|
2887
|
+
include_chrome=False, # chrome://newtab, chrome://settings, etc. are not valid frames we can control (for sanity reasons)
|
|
2888
|
+
include_chrome_extensions=False, # chrome-extension://
|
|
2889
|
+
include_chrome_error=False, # chrome-error:// (e.g. when iframes fail to load or are blocked by uBlock Origin)
|
|
2890
|
+
),
|
|
2891
|
+
}
|
|
2892
|
+
|
|
2893
|
+
# Check if frame is cross-origin based on crossOriginIsolatedContextType
|
|
2894
|
+
cross_origin_type = frame.get('crossOriginIsolatedContextType')
|
|
2895
|
+
if cross_origin_type and cross_origin_type != 'NotIsolated':
|
|
2896
|
+
frame_info['isCrossOrigin'] = True
|
|
2897
|
+
|
|
2898
|
+
# For iframe targets, the frame itself is likely cross-origin
|
|
2899
|
+
if target.get('type') == 'iframe':
|
|
2900
|
+
frame_info['isCrossOrigin'] = True
|
|
2901
|
+
|
|
2902
|
+
# Skip cross-origin frames if support is disabled
|
|
2903
|
+
if not include_cross_origin and frame_info.get('isCrossOrigin'):
|
|
2904
|
+
return # Skip this frame and its children
|
|
2905
|
+
|
|
2906
|
+
# Add child frame IDs (note: OOPIFs won't appear here)
|
|
2907
|
+
child_frames = node.get('childFrames', [])
|
|
2908
|
+
for child in child_frames:
|
|
2909
|
+
child_frame = child.get('frame', {})
|
|
2910
|
+
child_frame_id = child_frame.get('id')
|
|
2911
|
+
if child_frame_id:
|
|
2912
|
+
frame_info['childFrameIds'].append(child_frame_id)
|
|
2913
|
+
|
|
2914
|
+
# Store or merge frame info
|
|
2915
|
+
if current_frame_id in all_frames:
|
|
2916
|
+
# Frame already seen from another target, merge info
|
|
2917
|
+
existing = all_frames[current_frame_id]
|
|
2918
|
+
# If this is an iframe target, it has direct access to the frame
|
|
2919
|
+
if target.get('type') == 'iframe':
|
|
2920
|
+
existing['frameTargetId'] = target_id
|
|
2921
|
+
existing['isCrossOrigin'] = True
|
|
2922
|
+
else:
|
|
2923
|
+
all_frames[current_frame_id] = frame_info
|
|
2924
|
+
|
|
2925
|
+
# Process child frames recursively (only if we're not skipping this frame)
|
|
2926
|
+
if include_cross_origin or not frame_info.get('isCrossOrigin'):
|
|
2927
|
+
for child in child_frames:
|
|
2928
|
+
process_frame_tree(child, current_frame_id)
|
|
2929
|
+
|
|
2930
|
+
# Process the entire frame tree
|
|
2931
|
+
process_frame_tree(frame_tree_result.get('frameTree', {}))
|
|
2932
|
+
|
|
2933
|
+
except Exception as e:
|
|
2934
|
+
# Target doesn't support Page domain or has no frames
|
|
2935
|
+
self.logger.debug(f'Failed to get frame tree for target {target_id}: {e}')
|
|
2936
|
+
|
|
2937
|
+
# Second pass: populate backend node IDs and parent target IDs
|
|
2938
|
+
# Only do this if cross-origin support is enabled
|
|
2939
|
+
if include_cross_origin:
|
|
2940
|
+
await self._populate_frame_metadata(all_frames, target_sessions)
|
|
2941
|
+
|
|
2942
|
+
return all_frames, target_sessions
|
|
2943
|
+
|
|
2944
|
+
async def _populate_frame_metadata(self, all_frames: dict[str, dict], target_sessions: dict[str, str]) -> None:
|
|
2945
|
+
"""Populate additional frame metadata like backend node IDs and parent target IDs.
|
|
2946
|
+
|
|
2947
|
+
Args:
|
|
2948
|
+
all_frames: Frame hierarchy dict to populate
|
|
2949
|
+
target_sessions: Active target sessions
|
|
2950
|
+
"""
|
|
2951
|
+
for frame_id_iter, frame_info in all_frames.items():
|
|
2952
|
+
parent_frame_id = frame_info.get('parentFrameId')
|
|
2953
|
+
|
|
2954
|
+
if parent_frame_id and parent_frame_id in all_frames:
|
|
2955
|
+
parent_frame_info = all_frames[parent_frame_id]
|
|
2956
|
+
parent_target_id = parent_frame_info.get('frameTargetId')
|
|
2957
|
+
|
|
2958
|
+
# Store parent target ID
|
|
2959
|
+
frame_info['parentTargetId'] = parent_target_id
|
|
2960
|
+
|
|
2961
|
+
# Try to get backend node ID from parent context
|
|
2962
|
+
if parent_target_id in target_sessions:
|
|
2963
|
+
assert parent_target_id is not None
|
|
2964
|
+
parent_session_id = target_sessions[parent_target_id]
|
|
2965
|
+
try:
|
|
2966
|
+
# Enable DOM domain
|
|
2967
|
+
await self.cdp_client.send.DOM.enable(session_id=parent_session_id)
|
|
2968
|
+
|
|
2969
|
+
# Get frame owner info to find backend node ID
|
|
2970
|
+
frame_owner = await self.cdp_client.send.DOM.getFrameOwner(
|
|
2971
|
+
params={'frameId': frame_id_iter}, session_id=parent_session_id
|
|
2972
|
+
)
|
|
2973
|
+
|
|
2974
|
+
if frame_owner:
|
|
2975
|
+
frame_info['backendNodeId'] = frame_owner.get('backendNodeId')
|
|
2976
|
+
frame_info['nodeId'] = frame_owner.get('nodeId')
|
|
2977
|
+
|
|
2978
|
+
except Exception:
|
|
2979
|
+
# Frame owner not available (likely cross-origin)
|
|
2980
|
+
pass
|
|
2981
|
+
|
|
2982
|
+
async def find_frame_target(self, frame_id: str, all_frames: dict[str, dict] | None = None) -> dict | None:
|
|
2983
|
+
"""Find the frame info for a specific frame ID.
|
|
2984
|
+
|
|
2985
|
+
Args:
|
|
2986
|
+
frame_id: The frame ID to search for
|
|
2987
|
+
all_frames: Optional pre-built frame hierarchy. If None, will call get_all_frames()
|
|
2988
|
+
|
|
2989
|
+
Returns:
|
|
2990
|
+
Frame info dict if found, None otherwise
|
|
2991
|
+
"""
|
|
2992
|
+
if all_frames is None:
|
|
2993
|
+
all_frames, _ = await self.get_all_frames()
|
|
2994
|
+
|
|
2995
|
+
return all_frames.get(frame_id)
|
|
2996
|
+
|
|
2997
|
+
async def cdp_client_for_target(self, target_id: TargetID) -> CDPSession:
|
|
2998
|
+
return await self.get_or_create_cdp_session(target_id, focus=False)
|
|
2999
|
+
|
|
3000
|
+
def get_target_id_from_session_id(self, session_id: SessionID | None) -> TargetID | None:
|
|
3001
|
+
"""Look up target_id from a CDP session_id.
|
|
3002
|
+
|
|
3003
|
+
Args:
|
|
3004
|
+
session_id: The CDP session ID to look up
|
|
3005
|
+
|
|
3006
|
+
Returns:
|
|
3007
|
+
The target_id for this session, or None if not found
|
|
3008
|
+
"""
|
|
3009
|
+
if not session_id:
|
|
3010
|
+
return None
|
|
3011
|
+
for cdp_session in self._cdp_session_pool.values():
|
|
3012
|
+
if cdp_session.session_id == session_id:
|
|
3013
|
+
return cdp_session.target_id
|
|
3014
|
+
return None
|
|
3015
|
+
|
|
3016
|
+
async def cdp_client_for_frame(self, frame_id: str) -> CDPSession:
|
|
3017
|
+
"""Get a CDP client attached to the target containing the specified frame.
|
|
3018
|
+
|
|
3019
|
+
Builds a unified frame hierarchy from all targets to find the correct target
|
|
3020
|
+
for any frame, including OOPIFs (Out-of-Process iframes).
|
|
3021
|
+
|
|
3022
|
+
Args:
|
|
3023
|
+
frame_id: The frame ID to search for
|
|
3024
|
+
|
|
3025
|
+
Returns:
|
|
3026
|
+
Tuple of (cdp_cdp_session, target_id) for the target containing the frame
|
|
3027
|
+
|
|
3028
|
+
Raises:
|
|
3029
|
+
ValueError: If the frame is not found in any target
|
|
3030
|
+
"""
|
|
3031
|
+
# If cross-origin iframes are disabled, just use the main session
|
|
3032
|
+
if not self.browser_profile.cross_origin_iframes:
|
|
3033
|
+
return await self.get_or_create_cdp_session()
|
|
3034
|
+
|
|
3035
|
+
# Get complete frame hierarchy
|
|
3036
|
+
all_frames, target_sessions = await self.get_all_frames()
|
|
3037
|
+
|
|
3038
|
+
# Find the requested frame
|
|
3039
|
+
frame_info = await self.find_frame_target(frame_id, all_frames)
|
|
3040
|
+
|
|
3041
|
+
if frame_info:
|
|
3042
|
+
target_id = frame_info.get('frameTargetId')
|
|
3043
|
+
|
|
3044
|
+
if target_id in target_sessions:
|
|
3045
|
+
assert target_id is not None
|
|
3046
|
+
# Use existing session
|
|
3047
|
+
session_id = target_sessions[target_id]
|
|
3048
|
+
# Return the client with session attached (don't change focus)
|
|
3049
|
+
return await self.get_or_create_cdp_session(target_id, focus=False)
|
|
3050
|
+
|
|
3051
|
+
# Frame not found
|
|
3052
|
+
raise ValueError(f"Frame with ID '{frame_id}' not found in any target")
|
|
3053
|
+
|
|
3054
|
+
async def cdp_client_for_node(self, node: EnhancedDOMTreeNode) -> CDPSession:
|
|
3055
|
+
"""Get CDP client for a specific DOM node based on its frame.
|
|
3056
|
+
|
|
3057
|
+
IMPORTANT: backend_node_id is only valid in the session where the DOM was captured.
|
|
3058
|
+
We trust the node's session_id/frame_id/target_id instead of searching all sessions.
|
|
3059
|
+
"""
|
|
3060
|
+
|
|
3061
|
+
# Strategy 1: If node has session_id, try to use that exact session (most specific)
|
|
3062
|
+
if node.session_id:
|
|
3063
|
+
try:
|
|
3064
|
+
# Find the CDP session by session_id
|
|
3065
|
+
for cdp_session in self._cdp_session_pool.values():
|
|
3066
|
+
if cdp_session.session_id == node.session_id:
|
|
3067
|
+
self.logger.debug(
|
|
3068
|
+
f'✅ Using session from node.session_id for node {node.backend_node_id}: {cdp_session.url}'
|
|
3069
|
+
)
|
|
3070
|
+
return cdp_session
|
|
3071
|
+
except Exception as e:
|
|
3072
|
+
self.logger.debug(f'Failed to get session by session_id {node.session_id}: {e}')
|
|
3073
|
+
|
|
3074
|
+
# Strategy 2: If node has frame_id, use that frame's session
|
|
3075
|
+
if node.frame_id:
|
|
3076
|
+
try:
|
|
3077
|
+
cdp_session = await self.cdp_client_for_frame(node.frame_id)
|
|
3078
|
+
self.logger.debug(f'✅ Using session from node.frame_id for node {node.backend_node_id}: {cdp_session.url}')
|
|
3079
|
+
return cdp_session
|
|
3080
|
+
except Exception as e:
|
|
3081
|
+
self.logger.debug(f'Failed to get session for frame {node.frame_id}: {e}')
|
|
3082
|
+
|
|
3083
|
+
# Strategy 3: If node has target_id, use that target's session
|
|
3084
|
+
if node.target_id:
|
|
3085
|
+
try:
|
|
3086
|
+
cdp_session = await self.get_or_create_cdp_session(target_id=node.target_id, focus=False)
|
|
3087
|
+
self.logger.debug(f'✅ Using session from node.target_id for node {node.backend_node_id}: {cdp_session.url}')
|
|
3088
|
+
return cdp_session
|
|
3089
|
+
except Exception as e:
|
|
3090
|
+
self.logger.debug(f'Failed to get session for target {node.target_id}: {e}')
|
|
3091
|
+
|
|
3092
|
+
# Strategy 4: Fallback to agent_focus (the page where agent is currently working)
|
|
3093
|
+
if self.agent_focus:
|
|
3094
|
+
self.logger.warning(
|
|
3095
|
+
f'⚠️ Node {node.backend_node_id} has no session/frame/target info. '
|
|
3096
|
+
f'Using agent_focus session: {self.agent_focus.url}'
|
|
3097
|
+
)
|
|
3098
|
+
return self.agent_focus
|
|
3099
|
+
|
|
3100
|
+
# Last resort: use main session
|
|
3101
|
+
self.logger.error(f'❌ No session info for node {node.backend_node_id} and no agent_focus available. Using main session.')
|
|
3102
|
+
return await self.get_or_create_cdp_session()
|
|
3103
|
+
|
|
3104
|
+
@observe_debug(ignore_input=True, ignore_output=True, name='take_screenshot')
|
|
3105
|
+
async def take_screenshot(
|
|
3106
|
+
self,
|
|
3107
|
+
path: str | None = None,
|
|
3108
|
+
full_page: bool = False,
|
|
3109
|
+
format: str = 'png',
|
|
3110
|
+
quality: int | None = None,
|
|
3111
|
+
clip: dict | None = None,
|
|
3112
|
+
) -> bytes:
|
|
3113
|
+
"""Take a screenshot using CDP.
|
|
3114
|
+
|
|
3115
|
+
Args:
|
|
3116
|
+
path: Optional file path to save screenshot
|
|
3117
|
+
full_page: Capture entire scrollable page beyond viewport
|
|
3118
|
+
format: Image format ('png', 'jpeg', 'webp')
|
|
3119
|
+
quality: Quality 0-100 for JPEG format
|
|
3120
|
+
clip: Region to capture {'x': int, 'y': int, 'width': int, 'height': int}
|
|
3121
|
+
|
|
3122
|
+
Returns:
|
|
3123
|
+
Screenshot data as bytes
|
|
3124
|
+
"""
|
|
3125
|
+
import base64
|
|
3126
|
+
|
|
3127
|
+
from cdp_use.cdp.page import CaptureScreenshotParameters
|
|
3128
|
+
|
|
3129
|
+
cdp_session = await self.get_or_create_cdp_session()
|
|
3130
|
+
|
|
3131
|
+
# Build parameters dict explicitly to satisfy TypedDict expectations
|
|
3132
|
+
params: CaptureScreenshotParameters = {
|
|
3133
|
+
'format': format,
|
|
3134
|
+
'captureBeyondViewport': full_page,
|
|
3135
|
+
}
|
|
3136
|
+
|
|
3137
|
+
if quality is not None and format == 'jpeg':
|
|
3138
|
+
params['quality'] = quality
|
|
3139
|
+
|
|
3140
|
+
if clip:
|
|
3141
|
+
params['clip'] = {
|
|
3142
|
+
'x': clip['x'],
|
|
3143
|
+
'y': clip['y'],
|
|
3144
|
+
'width': clip['width'],
|
|
3145
|
+
'height': clip['height'],
|
|
3146
|
+
'scale': 1,
|
|
3147
|
+
}
|
|
3148
|
+
|
|
3149
|
+
params = CaptureScreenshotParameters(**params)
|
|
3150
|
+
|
|
3151
|
+
result = await cdp_session.cdp_client.send.Page.captureScreenshot(params=params, session_id=cdp_session.session_id)
|
|
3152
|
+
|
|
3153
|
+
if not result or 'data' not in result:
|
|
3154
|
+
raise Exception('Screenshot failed - no data returned')
|
|
3155
|
+
|
|
3156
|
+
screenshot_data = base64.b64decode(result['data'])
|
|
3157
|
+
|
|
3158
|
+
if path:
|
|
3159
|
+
Path(path).write_bytes(screenshot_data)
|
|
3160
|
+
|
|
3161
|
+
return screenshot_data
|
|
3162
|
+
|
|
3163
|
+
async def screenshot_element(
|
|
3164
|
+
self,
|
|
3165
|
+
selector: str,
|
|
3166
|
+
path: str | None = None,
|
|
3167
|
+
format: str = 'png',
|
|
3168
|
+
quality: int | None = None,
|
|
3169
|
+
) -> bytes:
|
|
3170
|
+
"""Take a screenshot of a specific element.
|
|
3171
|
+
|
|
3172
|
+
Args:
|
|
3173
|
+
selector: CSS selector for the element
|
|
3174
|
+
path: Optional file path to save screenshot
|
|
3175
|
+
format: Image format ('png', 'jpeg', 'webp')
|
|
3176
|
+
quality: Quality 0-100 for JPEG format
|
|
3177
|
+
|
|
3178
|
+
Returns:
|
|
3179
|
+
Screenshot data as bytes
|
|
3180
|
+
"""
|
|
3181
|
+
|
|
3182
|
+
bounds = await self._get_element_bounds(selector)
|
|
3183
|
+
if not bounds:
|
|
3184
|
+
raise ValueError(f"Element '{selector}' not found or has no bounds")
|
|
3185
|
+
|
|
3186
|
+
return await self.take_screenshot(
|
|
3187
|
+
path=path,
|
|
3188
|
+
format=format,
|
|
3189
|
+
quality=quality,
|
|
3190
|
+
clip=bounds,
|
|
3191
|
+
)
|
|
3192
|
+
|
|
3193
|
+
async def _get_element_bounds(self, selector: str) -> dict | None:
|
|
3194
|
+
"""Get element bounding box using CDP."""
|
|
3195
|
+
|
|
3196
|
+
cdp_session = await self.get_or_create_cdp_session()
|
|
3197
|
+
|
|
3198
|
+
# Get document
|
|
3199
|
+
doc = await cdp_session.cdp_client.send.DOM.getDocument(params={'depth': 1}, session_id=cdp_session.session_id)
|
|
3200
|
+
|
|
3201
|
+
# Query selector
|
|
3202
|
+
node_result = await cdp_session.cdp_client.send.DOM.querySelector(
|
|
3203
|
+
params={'nodeId': doc['root']['nodeId'], 'selector': selector}, session_id=cdp_session.session_id
|
|
3204
|
+
)
|
|
3205
|
+
|
|
3206
|
+
node_id = node_result.get('nodeId')
|
|
3207
|
+
if not node_id:
|
|
3208
|
+
return None
|
|
3209
|
+
|
|
3210
|
+
# Get bounding box
|
|
3211
|
+
box_result = await cdp_session.cdp_client.send.DOM.getBoxModel(
|
|
3212
|
+
params={'nodeId': node_id}, session_id=cdp_session.session_id
|
|
3213
|
+
)
|
|
3214
|
+
|
|
3215
|
+
box_model = box_result.get('model')
|
|
3216
|
+
if not box_model:
|
|
3217
|
+
return None
|
|
3218
|
+
|
|
3219
|
+
content = box_model['content']
|
|
3220
|
+
return {
|
|
3221
|
+
'x': min(content[0], content[2], content[4], content[6]),
|
|
3222
|
+
'y': min(content[1], content[3], content[5], content[7]),
|
|
3223
|
+
'width': max(content[0], content[2], content[4], content[6]) - min(content[0], content[2], content[4], content[6]),
|
|
3224
|
+
'height': max(content[1], content[3], content[5], content[7]) - min(content[1], content[3], content[5], content[7]),
|
|
3225
|
+
}
|