optexity-browser-use 0.9.5__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (147) hide show
  1. browser_use/__init__.py +157 -0
  2. browser_use/actor/__init__.py +11 -0
  3. browser_use/actor/element.py +1175 -0
  4. browser_use/actor/mouse.py +134 -0
  5. browser_use/actor/page.py +561 -0
  6. browser_use/actor/playground/flights.py +41 -0
  7. browser_use/actor/playground/mixed_automation.py +54 -0
  8. browser_use/actor/playground/playground.py +236 -0
  9. browser_use/actor/utils.py +176 -0
  10. browser_use/agent/cloud_events.py +282 -0
  11. browser_use/agent/gif.py +424 -0
  12. browser_use/agent/judge.py +170 -0
  13. browser_use/agent/message_manager/service.py +473 -0
  14. browser_use/agent/message_manager/utils.py +52 -0
  15. browser_use/agent/message_manager/views.py +98 -0
  16. browser_use/agent/prompts.py +413 -0
  17. browser_use/agent/service.py +2316 -0
  18. browser_use/agent/system_prompt.md +185 -0
  19. browser_use/agent/system_prompt_flash.md +10 -0
  20. browser_use/agent/system_prompt_no_thinking.md +183 -0
  21. browser_use/agent/views.py +743 -0
  22. browser_use/browser/__init__.py +41 -0
  23. browser_use/browser/cloud/cloud.py +203 -0
  24. browser_use/browser/cloud/views.py +89 -0
  25. browser_use/browser/events.py +578 -0
  26. browser_use/browser/profile.py +1158 -0
  27. browser_use/browser/python_highlights.py +548 -0
  28. browser_use/browser/session.py +3225 -0
  29. browser_use/browser/session_manager.py +399 -0
  30. browser_use/browser/video_recorder.py +162 -0
  31. browser_use/browser/views.py +200 -0
  32. browser_use/browser/watchdog_base.py +260 -0
  33. browser_use/browser/watchdogs/__init__.py +0 -0
  34. browser_use/browser/watchdogs/aboutblank_watchdog.py +253 -0
  35. browser_use/browser/watchdogs/crash_watchdog.py +335 -0
  36. browser_use/browser/watchdogs/default_action_watchdog.py +2729 -0
  37. browser_use/browser/watchdogs/dom_watchdog.py +817 -0
  38. browser_use/browser/watchdogs/downloads_watchdog.py +1277 -0
  39. browser_use/browser/watchdogs/local_browser_watchdog.py +461 -0
  40. browser_use/browser/watchdogs/permissions_watchdog.py +43 -0
  41. browser_use/browser/watchdogs/popups_watchdog.py +143 -0
  42. browser_use/browser/watchdogs/recording_watchdog.py +126 -0
  43. browser_use/browser/watchdogs/screenshot_watchdog.py +62 -0
  44. browser_use/browser/watchdogs/security_watchdog.py +280 -0
  45. browser_use/browser/watchdogs/storage_state_watchdog.py +335 -0
  46. browser_use/cli.py +2359 -0
  47. browser_use/code_use/__init__.py +16 -0
  48. browser_use/code_use/formatting.py +192 -0
  49. browser_use/code_use/namespace.py +665 -0
  50. browser_use/code_use/notebook_export.py +276 -0
  51. browser_use/code_use/service.py +1340 -0
  52. browser_use/code_use/system_prompt.md +574 -0
  53. browser_use/code_use/utils.py +150 -0
  54. browser_use/code_use/views.py +171 -0
  55. browser_use/config.py +505 -0
  56. browser_use/controller/__init__.py +3 -0
  57. browser_use/dom/enhanced_snapshot.py +161 -0
  58. browser_use/dom/markdown_extractor.py +169 -0
  59. browser_use/dom/playground/extraction.py +312 -0
  60. browser_use/dom/playground/multi_act.py +32 -0
  61. browser_use/dom/serializer/clickable_elements.py +200 -0
  62. browser_use/dom/serializer/code_use_serializer.py +287 -0
  63. browser_use/dom/serializer/eval_serializer.py +478 -0
  64. browser_use/dom/serializer/html_serializer.py +212 -0
  65. browser_use/dom/serializer/paint_order.py +197 -0
  66. browser_use/dom/serializer/serializer.py +1170 -0
  67. browser_use/dom/service.py +825 -0
  68. browser_use/dom/utils.py +129 -0
  69. browser_use/dom/views.py +906 -0
  70. browser_use/exceptions.py +5 -0
  71. browser_use/filesystem/__init__.py +0 -0
  72. browser_use/filesystem/file_system.py +619 -0
  73. browser_use/init_cmd.py +376 -0
  74. browser_use/integrations/gmail/__init__.py +24 -0
  75. browser_use/integrations/gmail/actions.py +115 -0
  76. browser_use/integrations/gmail/service.py +225 -0
  77. browser_use/llm/__init__.py +155 -0
  78. browser_use/llm/anthropic/chat.py +242 -0
  79. browser_use/llm/anthropic/serializer.py +312 -0
  80. browser_use/llm/aws/__init__.py +36 -0
  81. browser_use/llm/aws/chat_anthropic.py +242 -0
  82. browser_use/llm/aws/chat_bedrock.py +289 -0
  83. browser_use/llm/aws/serializer.py +257 -0
  84. browser_use/llm/azure/chat.py +91 -0
  85. browser_use/llm/base.py +57 -0
  86. browser_use/llm/browser_use/__init__.py +3 -0
  87. browser_use/llm/browser_use/chat.py +201 -0
  88. browser_use/llm/cerebras/chat.py +193 -0
  89. browser_use/llm/cerebras/serializer.py +109 -0
  90. browser_use/llm/deepseek/chat.py +212 -0
  91. browser_use/llm/deepseek/serializer.py +109 -0
  92. browser_use/llm/exceptions.py +29 -0
  93. browser_use/llm/google/__init__.py +3 -0
  94. browser_use/llm/google/chat.py +542 -0
  95. browser_use/llm/google/serializer.py +120 -0
  96. browser_use/llm/groq/chat.py +229 -0
  97. browser_use/llm/groq/parser.py +158 -0
  98. browser_use/llm/groq/serializer.py +159 -0
  99. browser_use/llm/messages.py +238 -0
  100. browser_use/llm/models.py +271 -0
  101. browser_use/llm/oci_raw/__init__.py +10 -0
  102. browser_use/llm/oci_raw/chat.py +443 -0
  103. browser_use/llm/oci_raw/serializer.py +229 -0
  104. browser_use/llm/ollama/chat.py +97 -0
  105. browser_use/llm/ollama/serializer.py +143 -0
  106. browser_use/llm/openai/chat.py +264 -0
  107. browser_use/llm/openai/like.py +15 -0
  108. browser_use/llm/openai/serializer.py +165 -0
  109. browser_use/llm/openrouter/chat.py +211 -0
  110. browser_use/llm/openrouter/serializer.py +26 -0
  111. browser_use/llm/schema.py +176 -0
  112. browser_use/llm/views.py +48 -0
  113. browser_use/logging_config.py +330 -0
  114. browser_use/mcp/__init__.py +18 -0
  115. browser_use/mcp/__main__.py +12 -0
  116. browser_use/mcp/client.py +544 -0
  117. browser_use/mcp/controller.py +264 -0
  118. browser_use/mcp/server.py +1114 -0
  119. browser_use/observability.py +204 -0
  120. browser_use/py.typed +0 -0
  121. browser_use/sandbox/__init__.py +41 -0
  122. browser_use/sandbox/sandbox.py +637 -0
  123. browser_use/sandbox/views.py +132 -0
  124. browser_use/screenshots/__init__.py +1 -0
  125. browser_use/screenshots/service.py +52 -0
  126. browser_use/sync/__init__.py +6 -0
  127. browser_use/sync/auth.py +357 -0
  128. browser_use/sync/service.py +161 -0
  129. browser_use/telemetry/__init__.py +51 -0
  130. browser_use/telemetry/service.py +112 -0
  131. browser_use/telemetry/views.py +101 -0
  132. browser_use/tokens/__init__.py +0 -0
  133. browser_use/tokens/custom_pricing.py +24 -0
  134. browser_use/tokens/mappings.py +4 -0
  135. browser_use/tokens/service.py +580 -0
  136. browser_use/tokens/views.py +108 -0
  137. browser_use/tools/registry/service.py +572 -0
  138. browser_use/tools/registry/views.py +174 -0
  139. browser_use/tools/service.py +1675 -0
  140. browser_use/tools/utils.py +82 -0
  141. browser_use/tools/views.py +100 -0
  142. browser_use/utils.py +670 -0
  143. optexity_browser_use-0.9.5.dist-info/METADATA +344 -0
  144. optexity_browser_use-0.9.5.dist-info/RECORD +147 -0
  145. optexity_browser_use-0.9.5.dist-info/WHEEL +4 -0
  146. optexity_browser_use-0.9.5.dist-info/entry_points.txt +3 -0
  147. optexity_browser_use-0.9.5.dist-info/licenses/LICENSE +21 -0
@@ -0,0 +1,2316 @@
1
+ import asyncio
2
+ import gc
3
+ import inspect
4
+ import json
5
+ import logging
6
+ import re
7
+ import tempfile
8
+ import time
9
+ from collections.abc import Awaitable, Callable
10
+ from pathlib import Path
11
+ from typing import Any, Generic, Literal, TypeVar
12
+ from urllib.parse import urlparse
13
+
14
+ from dotenv import load_dotenv
15
+
16
+ from browser_use.agent.cloud_events import (
17
+ CreateAgentOutputFileEvent,
18
+ CreateAgentSessionEvent,
19
+ CreateAgentStepEvent,
20
+ CreateAgentTaskEvent,
21
+ UpdateAgentTaskEvent,
22
+ )
23
+ from browser_use.agent.message_manager.utils import save_conversation
24
+ from browser_use.llm.base import BaseChatModel
25
+ from browser_use.llm.messages import BaseMessage, ContentPartImageParam, ContentPartTextParam, UserMessage
26
+ from browser_use.tokens.service import TokenCost
27
+
28
+ load_dotenv()
29
+
30
+ from bubus import EventBus
31
+ from pydantic import BaseModel, ValidationError
32
+ from uuid_extensions import uuid7str
33
+
34
+ from browser_use import Browser, BrowserProfile, BrowserSession
35
+ from browser_use.agent.judge import construct_judge_messages
36
+
37
+ # Lazy import for gif to avoid heavy agent.views import at startup
38
+ # from browser_use.agent.gif import create_history_gif
39
+ from browser_use.agent.message_manager.service import (
40
+ MessageManager,
41
+ )
42
+ from browser_use.agent.prompts import SystemPrompt
43
+ from browser_use.agent.views import (
44
+ ActionResult,
45
+ AgentError,
46
+ AgentHistory,
47
+ AgentHistoryList,
48
+ AgentOutput,
49
+ AgentSettings,
50
+ AgentState,
51
+ AgentStepInfo,
52
+ AgentStructuredOutput,
53
+ BrowserStateHistory,
54
+ JudgementResult,
55
+ StepMetadata,
56
+ )
57
+ from browser_use.browser.session import DEFAULT_BROWSER_PROFILE
58
+ from browser_use.browser.views import BrowserStateSummary
59
+ from browser_use.config import CONFIG
60
+ from browser_use.dom.views import DOMInteractedElement
61
+ from browser_use.filesystem.file_system import FileSystem
62
+ from browser_use.observability import observe, observe_debug
63
+ from browser_use.telemetry.service import ProductTelemetry
64
+ from browser_use.telemetry.views import AgentTelemetryEvent
65
+ from browser_use.tools.registry.views import ActionModel
66
+ from browser_use.tools.service import Tools
67
+ from browser_use.utils import (
68
+ URL_PATTERN,
69
+ _log_pretty_path,
70
+ check_latest_browser_use_version,
71
+ get_browser_use_version,
72
+ time_execution_async,
73
+ time_execution_sync,
74
+ )
75
+
76
+ logger = logging.getLogger(__name__)
77
+
78
+
79
+ def log_response(response: AgentOutput, registry=None, logger=None) -> None:
80
+ """Utility function to log the model's response."""
81
+
82
+ # Use module logger if no logger provided
83
+ if logger is None:
84
+ logger = logging.getLogger(__name__)
85
+
86
+ # Only log thinking if it's present
87
+ if response.current_state.thinking:
88
+ logger.debug(f'💡 Thinking:\n{response.current_state.thinking}')
89
+
90
+ # Only log evaluation if it's not empty
91
+ eval_goal = response.current_state.evaluation_previous_goal
92
+ if eval_goal:
93
+ if 'success' in eval_goal.lower():
94
+ emoji = '👍'
95
+ # Green color for success
96
+ logger.info(f' \033[32m{emoji} Eval: {eval_goal}\033[0m')
97
+ elif 'failure' in eval_goal.lower():
98
+ emoji = '⚠️'
99
+ # Red color for failure
100
+ logger.info(f' \033[31m{emoji} Eval: {eval_goal}\033[0m')
101
+ else:
102
+ emoji = '❔'
103
+ # No color for unknown/neutral
104
+ logger.info(f' {emoji} Eval: {eval_goal}')
105
+
106
+ # Always log memory if present
107
+ if response.current_state.memory:
108
+ logger.info(f' 🧠 Memory: {response.current_state.memory}')
109
+
110
+ # Only log next goal if it's not empty
111
+ next_goal = response.current_state.next_goal
112
+ if next_goal:
113
+ # Blue color for next goal
114
+ logger.info(f' \033[34m🎯 Next goal: {next_goal}\033[0m')
115
+
116
+
117
+ Context = TypeVar('Context')
118
+
119
+
120
+ AgentHookFunc = Callable[['Agent'], Awaitable[None]]
121
+
122
+
123
+ class Agent(Generic[Context, AgentStructuredOutput]):
124
+ @time_execution_sync('--init')
125
+ def __init__(
126
+ self,
127
+ task: str,
128
+ llm: BaseChatModel | None = None,
129
+ # Optional parameters
130
+ browser_profile: BrowserProfile | None = None,
131
+ browser_session: BrowserSession | None = None,
132
+ browser: Browser | None = None, # Alias for browser_session
133
+ tools: Tools[Context] | None = None,
134
+ controller: Tools[Context] | None = None, # Alias for tools
135
+ # Initial agent run parameters
136
+ sensitive_data: dict[str, str | dict[str, str]] | None = None,
137
+ initial_actions: list[dict[str, dict[str, Any]]] | None = None,
138
+ # Cloud Callbacks
139
+ register_new_step_callback: (
140
+ Callable[['BrowserStateSummary', 'AgentOutput', int], None] # Sync callback
141
+ | Callable[['BrowserStateSummary', 'AgentOutput', int], Awaitable[None]] # Async callback
142
+ | None
143
+ ) = None,
144
+ register_done_callback: (
145
+ Callable[['AgentHistoryList'], Awaitable[None]] # Async Callback
146
+ | Callable[['AgentHistoryList'], None] # Sync Callback
147
+ | None
148
+ ) = None,
149
+ register_external_agent_status_raise_error_callback: Callable[[], Awaitable[bool]] | None = None,
150
+ register_should_stop_callback: Callable[[], Awaitable[bool]] | None = None,
151
+ # Agent settings
152
+ output_model_schema: type[AgentStructuredOutput] | None = None,
153
+ use_vision: bool | Literal['auto'] = 'auto',
154
+ save_conversation_path: str | Path | None = None,
155
+ save_conversation_path_encoding: str | None = 'utf-8',
156
+ max_failures: int = 3,
157
+ override_system_message: str | None = None,
158
+ extend_system_message: str | None = None,
159
+ generate_gif: bool | str = False,
160
+ available_file_paths: list[str] | None = None,
161
+ include_attributes: list[str] | None = None,
162
+ max_actions_per_step: int = 10,
163
+ use_thinking: bool = True,
164
+ flash_mode: bool = False,
165
+ max_history_items: int | None = None,
166
+ page_extraction_llm: BaseChatModel | None = None,
167
+ use_judge: bool = True,
168
+ judge_llm: BaseChatModel | None = None,
169
+ injected_agent_state: AgentState | None = None,
170
+ source: str | None = None,
171
+ file_system_path: str | None = None,
172
+ task_id: str | None = None,
173
+ calculate_cost: bool = False,
174
+ display_files_in_done_text: bool = True,
175
+ include_tool_call_examples: bool = False,
176
+ vision_detail_level: Literal['auto', 'low', 'high'] = 'auto',
177
+ llm_timeout: int | None = None,
178
+ step_timeout: int = 120,
179
+ directly_open_url: bool = True,
180
+ include_recent_events: bool = False,
181
+ sample_images: list[ContentPartTextParam | ContentPartImageParam] | None = None,
182
+ final_response_after_failure: bool = True,
183
+ _url_shortening_limit: int = 25,
184
+ **kwargs,
185
+ ):
186
+ if llm is None:
187
+ default_llm_name = CONFIG.DEFAULT_LLM
188
+ if default_llm_name:
189
+ from browser_use.llm.models import get_llm_by_name
190
+
191
+ llm = get_llm_by_name(default_llm_name)
192
+ else:
193
+ # No default LLM specified, use the original default
194
+ from browser_use import ChatBrowserUse
195
+
196
+ llm = ChatBrowserUse()
197
+
198
+ # set flashmode = True if llm is ChatBrowserUse
199
+ if llm.provider == 'browser-use':
200
+ flash_mode = True
201
+
202
+ if page_extraction_llm is None:
203
+ page_extraction_llm = llm
204
+ if judge_llm is None:
205
+ judge_llm = llm
206
+ if available_file_paths is None:
207
+ available_file_paths = []
208
+
209
+ # Set timeout based on model name if not explicitly provided
210
+ if llm_timeout is None:
211
+
212
+ def _get_model_timeout(llm_model: BaseChatModel) -> int:
213
+ """Determine timeout based on model name"""
214
+ model_name = getattr(llm_model, 'model', '').lower()
215
+ if 'gemini' in model_name:
216
+ return 45
217
+ elif 'groq' in model_name:
218
+ return 30
219
+ elif 'o3' in model_name or 'claude' in model_name or 'sonnet' in model_name or 'deepseek' in model_name:
220
+ return 90
221
+ else:
222
+ return 60 # Default timeout
223
+
224
+ llm_timeout = _get_model_timeout(llm)
225
+
226
+ self.id = task_id or uuid7str()
227
+ self.task_id: str = self.id
228
+ self.session_id: str = uuid7str()
229
+
230
+ browser_profile = browser_profile or DEFAULT_BROWSER_PROFILE
231
+
232
+ # Handle browser vs browser_session parameter (browser takes precedence)
233
+ if browser and browser_session:
234
+ raise ValueError('Cannot specify both "browser" and "browser_session" parameters. Use "browser" for the cleaner API.')
235
+ browser_session = browser or browser_session
236
+
237
+ self.browser_session = browser_session or BrowserSession(
238
+ browser_profile=browser_profile,
239
+ id=uuid7str()[:-4] + self.id[-4:], # re-use the same 4-char suffix so they show up together in logs
240
+ )
241
+
242
+ # Initialize available file paths as direct attribute
243
+ self.available_file_paths = available_file_paths
244
+
245
+ # Core components
246
+ self.task = self._enhance_task_with_schema(task, output_model_schema)
247
+ self.llm = llm
248
+ self.judge_llm = judge_llm
249
+ self.directly_open_url = directly_open_url
250
+ self.include_recent_events = include_recent_events
251
+ self._url_shortening_limit = _url_shortening_limit
252
+ if tools is not None:
253
+ self.tools = tools
254
+ elif controller is not None:
255
+ self.tools = controller
256
+ else:
257
+ # Exclude screenshot tool when use_vision=False
258
+ exclude_actions = ['screenshot'] if use_vision is False else []
259
+ self.tools = Tools(exclude_actions=exclude_actions, display_files_in_done_text=display_files_in_done_text)
260
+
261
+ # Structured output
262
+ self.output_model_schema = output_model_schema
263
+ if self.output_model_schema is not None:
264
+ self.tools.use_structured_output_action(self.output_model_schema)
265
+
266
+ self.sensitive_data = sensitive_data
267
+
268
+ self.sample_images = sample_images
269
+
270
+ self.settings = AgentSettings(
271
+ use_vision=use_vision,
272
+ vision_detail_level=vision_detail_level,
273
+ save_conversation_path=save_conversation_path,
274
+ save_conversation_path_encoding=save_conversation_path_encoding,
275
+ max_failures=max_failures,
276
+ override_system_message=override_system_message,
277
+ extend_system_message=extend_system_message,
278
+ generate_gif=generate_gif,
279
+ include_attributes=include_attributes,
280
+ max_actions_per_step=max_actions_per_step,
281
+ use_thinking=use_thinking,
282
+ flash_mode=flash_mode,
283
+ max_history_items=max_history_items,
284
+ page_extraction_llm=page_extraction_llm,
285
+ calculate_cost=calculate_cost,
286
+ include_tool_call_examples=include_tool_call_examples,
287
+ llm_timeout=llm_timeout,
288
+ step_timeout=step_timeout,
289
+ final_response_after_failure=final_response_after_failure,
290
+ use_judge=use_judge,
291
+ )
292
+
293
+ # Token cost service
294
+ self.token_cost_service = TokenCost(include_cost=calculate_cost)
295
+ self.token_cost_service.register_llm(llm)
296
+ self.token_cost_service.register_llm(page_extraction_llm)
297
+ self.token_cost_service.register_llm(judge_llm)
298
+
299
+ # Initialize state
300
+ self.state = injected_agent_state or AgentState()
301
+
302
+ # Initialize history
303
+ self.history = AgentHistoryList(history=[], usage=None)
304
+
305
+ # Initialize agent directory
306
+ import time
307
+
308
+ timestamp = int(time.time())
309
+ base_tmp = Path(tempfile.gettempdir())
310
+ self.agent_directory = base_tmp / f'browser_use_agent_{self.id}_{timestamp}'
311
+
312
+ # Initialize file system and screenshot service
313
+ self._set_file_system(file_system_path)
314
+ self._set_screenshot_service()
315
+
316
+ # Action setup
317
+ self._setup_action_models()
318
+ self._set_browser_use_version_and_source(source)
319
+
320
+ initial_url = None
321
+
322
+ # only load url if no initial actions are provided
323
+ if self.directly_open_url and not self.state.follow_up_task and not initial_actions:
324
+ initial_url = self._extract_start_url(self.task)
325
+ if initial_url:
326
+ self.logger.info(f'🔗 Found URL in task: {initial_url}, adding as initial action...')
327
+ initial_actions = [{'navigate': {'url': initial_url, 'new_tab': False}}]
328
+
329
+ self.initial_url = initial_url
330
+
331
+ self.initial_actions = self._convert_initial_actions(initial_actions) if initial_actions else None
332
+ # Verify we can connect to the model
333
+ self._verify_and_setup_llm()
334
+
335
+ # TODO: move this logic to the LLMs
336
+ # Handle users trying to use use_vision=True with DeepSeek models
337
+ if 'deepseek' in self.llm.model.lower():
338
+ self.logger.warning('⚠️ DeepSeek models do not support use_vision=True yet. Setting use_vision=False for now...')
339
+ self.settings.use_vision = False
340
+
341
+ # Handle users trying to use use_vision=True with XAI models
342
+ if 'grok' in self.llm.model.lower():
343
+ self.logger.warning('⚠️ XAI models do not support use_vision=True yet. Setting use_vision=False for now...')
344
+ self.settings.use_vision = False
345
+
346
+ logger.debug(
347
+ f'{" +vision" if self.settings.use_vision else ""}'
348
+ f' extraction_model={self.settings.page_extraction_llm.model if self.settings.page_extraction_llm else "Unknown"}'
349
+ f'{" +file_system" if self.file_system else ""}'
350
+ )
351
+
352
+ # Initialize message manager with state
353
+ # Initial system prompt with all actions - will be updated during each step
354
+ self._message_manager = MessageManager(
355
+ task=self.task,
356
+ system_message=SystemPrompt(
357
+ max_actions_per_step=self.settings.max_actions_per_step,
358
+ override_system_message=override_system_message,
359
+ extend_system_message=extend_system_message,
360
+ use_thinking=self.settings.use_thinking,
361
+ flash_mode=self.settings.flash_mode,
362
+ ).get_system_message(),
363
+ file_system=self.file_system,
364
+ state=self.state.message_manager_state,
365
+ use_thinking=self.settings.use_thinking,
366
+ # Settings that were previously in MessageManagerSettings
367
+ include_attributes=self.settings.include_attributes,
368
+ sensitive_data=sensitive_data,
369
+ max_history_items=self.settings.max_history_items,
370
+ vision_detail_level=self.settings.vision_detail_level,
371
+ include_tool_call_examples=self.settings.include_tool_call_examples,
372
+ include_recent_events=self.include_recent_events,
373
+ sample_images=self.sample_images,
374
+ )
375
+
376
+ if self.sensitive_data:
377
+ # Check if sensitive_data has domain-specific credentials
378
+ has_domain_specific_credentials = any(isinstance(v, dict) for v in self.sensitive_data.values())
379
+
380
+ # If no allowed_domains are configured, show a security warning
381
+ if not self.browser_profile.allowed_domains:
382
+ self.logger.error(
383
+ '⚠️ Agent(sensitive_data=••••••••) was provided but Browser(allowed_domains=[...]) is not locked down! ⚠️\n'
384
+ ' ☠️ If the agent visits a malicious website and encounters a prompt-injection attack, your sensitive_data may be exposed!\n\n'
385
+ ' \n'
386
+ )
387
+
388
+ # If we're using domain-specific credentials, validate domain patterns
389
+ elif has_domain_specific_credentials:
390
+ # For domain-specific format, ensure all domain patterns are included in allowed_domains
391
+ domain_patterns = [k for k, v in self.sensitive_data.items() if isinstance(v, dict)]
392
+
393
+ # Validate each domain pattern against allowed_domains
394
+ for domain_pattern in domain_patterns:
395
+ is_allowed = False
396
+ for allowed_domain in self.browser_profile.allowed_domains:
397
+ # Special cases that don't require URL matching
398
+ if domain_pattern == allowed_domain or allowed_domain == '*':
399
+ is_allowed = True
400
+ break
401
+
402
+ # Need to create example URLs to compare the patterns
403
+ # Extract the domain parts, ignoring scheme
404
+ pattern_domain = domain_pattern.split('://')[-1] if '://' in domain_pattern else domain_pattern
405
+ allowed_domain_part = allowed_domain.split('://')[-1] if '://' in allowed_domain else allowed_domain
406
+
407
+ # Check if pattern is covered by an allowed domain
408
+ # Example: "google.com" is covered by "*.google.com"
409
+ if pattern_domain == allowed_domain_part or (
410
+ allowed_domain_part.startswith('*.')
411
+ and (
412
+ pattern_domain == allowed_domain_part[2:]
413
+ or pattern_domain.endswith('.' + allowed_domain_part[2:])
414
+ )
415
+ ):
416
+ is_allowed = True
417
+ break
418
+
419
+ if not is_allowed:
420
+ self.logger.warning(
421
+ f'⚠️ Domain pattern "{domain_pattern}" in sensitive_data is not covered by any pattern in allowed_domains={self.browser_profile.allowed_domains}\n'
422
+ f' This may be a security risk as credentials could be used on unintended domains.'
423
+ )
424
+
425
+ # Callbacks
426
+ self.register_new_step_callback = register_new_step_callback
427
+ self.register_done_callback = register_done_callback
428
+ self.register_should_stop_callback = register_should_stop_callback
429
+ self.register_external_agent_status_raise_error_callback = register_external_agent_status_raise_error_callback
430
+
431
+ # Telemetry
432
+ self.telemetry = ProductTelemetry()
433
+
434
+ # Event bus with WAL persistence
435
+ # Default to ~/.config/browseruse/events/{agent_session_id}.jsonl
436
+ # wal_path = CONFIG.BROWSER_USE_CONFIG_DIR / 'events' / f'{self.session_id}.jsonl'
437
+ self.eventbus = EventBus(name=f'Agent_{str(self.id)[-4:]}')
438
+
439
+ if self.settings.save_conversation_path:
440
+ self.settings.save_conversation_path = Path(self.settings.save_conversation_path).expanduser().resolve()
441
+ self.logger.info(f'💬 Saving conversation to {_log_pretty_path(self.settings.save_conversation_path)}')
442
+
443
+ # Initialize download tracking
444
+ assert self.browser_session is not None, 'BrowserSession is not set up'
445
+ self.has_downloads_path = self.browser_session.browser_profile.downloads_path is not None
446
+ if self.has_downloads_path:
447
+ self._last_known_downloads: list[str] = []
448
+ self.logger.debug('📁 Initialized download tracking for agent')
449
+
450
+ # Event-based pause control (kept out of AgentState for serialization)
451
+ self._external_pause_event = asyncio.Event()
452
+ self._external_pause_event.set()
453
+
454
+ def _enhance_task_with_schema(self, task: str, output_model_schema: type[AgentStructuredOutput] | None) -> str:
455
+ """Enhance task description with output schema information if provided."""
456
+ if output_model_schema is None:
457
+ return task
458
+
459
+ try:
460
+ schema = output_model_schema.model_json_schema()
461
+ import json
462
+
463
+ schema_json = json.dumps(schema, indent=2)
464
+
465
+ enhancement = f'\nExpected output format: {output_model_schema.__name__}\n{schema_json}'
466
+ return task + enhancement
467
+ except Exception as e:
468
+ self.logger.debug(f'Could not parse output schema: {e}')
469
+
470
+ return task
471
+
472
+ @property
473
+ def logger(self) -> logging.Logger:
474
+ """Get instance-specific logger with task ID in the name"""
475
+
476
+ _browser_session_id = self.browser_session.id if self.browser_session else '----'
477
+ _current_target_id = (
478
+ self.browser_session.agent_focus.target_id[-2:]
479
+ if self.browser_session and self.browser_session.agent_focus and self.browser_session.agent_focus.target_id
480
+ else '--'
481
+ )
482
+ return logging.getLogger(f'browser_use.Agent🅰 {self.task_id[-4:]} ⇢ 🅑 {_browser_session_id[-4:]} 🅣 {_current_target_id}')
483
+
484
+ @property
485
+ def browser_profile(self) -> BrowserProfile:
486
+ assert self.browser_session is not None, 'BrowserSession is not set up'
487
+ return self.browser_session.browser_profile
488
+
489
+ async def _check_and_update_downloads(self, context: str = '') -> None:
490
+ """Check for new downloads and update available file paths."""
491
+ if not self.has_downloads_path:
492
+ return
493
+
494
+ assert self.browser_session is not None, 'BrowserSession is not set up'
495
+
496
+ try:
497
+ current_downloads = self.browser_session.downloaded_files
498
+ if current_downloads != self._last_known_downloads:
499
+ self._update_available_file_paths(current_downloads)
500
+ self._last_known_downloads = current_downloads
501
+ if context:
502
+ self.logger.debug(f'📁 {context}: Updated available files')
503
+ except Exception as e:
504
+ error_context = f' {context}' if context else ''
505
+ self.logger.debug(f'📁 Failed to check for downloads{error_context}: {type(e).__name__}: {e}')
506
+
507
+ def _update_available_file_paths(self, downloads: list[str]) -> None:
508
+ """Update available_file_paths with downloaded files."""
509
+ if not self.has_downloads_path:
510
+ return
511
+
512
+ current_files = set(self.available_file_paths or [])
513
+ new_files = set(downloads) - current_files
514
+
515
+ if new_files:
516
+ self.available_file_paths = list(current_files | new_files)
517
+
518
+ self.logger.info(
519
+ f'📁 Added {len(new_files)} downloaded files to available_file_paths (total: {len(self.available_file_paths)} files)'
520
+ )
521
+ for file_path in new_files:
522
+ self.logger.info(f'📄 New file available: {file_path}')
523
+ else:
524
+ self.logger.debug(f'📁 No new downloads detected (tracking {len(current_files)} files)')
525
+
526
+ def _set_file_system(self, file_system_path: str | None = None) -> None:
527
+ # Check for conflicting parameters
528
+ if self.state.file_system_state and file_system_path:
529
+ raise ValueError(
530
+ 'Cannot provide both file_system_state (from agent state) and file_system_path. '
531
+ 'Either restore from existing state or create new file system at specified path, not both.'
532
+ )
533
+
534
+ # Check if we should restore from existing state first
535
+ if self.state.file_system_state:
536
+ try:
537
+ # Restore file system from state at the exact same location
538
+ self.file_system = FileSystem.from_state(self.state.file_system_state)
539
+ # The parent directory of base_dir is the original file_system_path
540
+ self.file_system_path = str(self.file_system.base_dir)
541
+ logger.debug(f'💾 File system restored from state to: {self.file_system_path}')
542
+ return
543
+ except Exception as e:
544
+ logger.error(f'💾 Failed to restore file system from state: {e}')
545
+ raise e
546
+
547
+ # Initialize new file system
548
+ try:
549
+ if file_system_path:
550
+ self.file_system = FileSystem(file_system_path)
551
+ self.file_system_path = file_system_path
552
+ else:
553
+ # Use the agent directory for file system
554
+ self.file_system = FileSystem(self.agent_directory)
555
+ self.file_system_path = str(self.agent_directory)
556
+ except Exception as e:
557
+ logger.error(f'💾 Failed to initialize file system: {e}.')
558
+ raise e
559
+
560
+ # Save file system state to agent state
561
+ self.state.file_system_state = self.file_system.get_state()
562
+
563
+ logger.debug(f'💾 File system path: {self.file_system_path}')
564
+
565
+ def _set_screenshot_service(self) -> None:
566
+ """Initialize screenshot service using agent directory"""
567
+ try:
568
+ from browser_use.screenshots.service import ScreenshotService
569
+
570
+ self.screenshot_service = ScreenshotService(self.agent_directory)
571
+ logger.debug(f'📸 Screenshot service initialized in: {self.agent_directory}/screenshots')
572
+ except Exception as e:
573
+ logger.error(f'📸 Failed to initialize screenshot service: {e}.')
574
+ raise e
575
+
576
+ def save_file_system_state(self) -> None:
577
+ """Save current file system state to agent state"""
578
+ if self.file_system:
579
+ self.state.file_system_state = self.file_system.get_state()
580
+ else:
581
+ logger.error('💾 File system is not set up. Cannot save state.')
582
+ raise ValueError('File system is not set up. Cannot save state.')
583
+
584
+ def _set_browser_use_version_and_source(self, source_override: str | None = None) -> None:
585
+ """Get the version from pyproject.toml and determine the source of the browser-use package"""
586
+ # Use the helper function for version detection
587
+ version = get_browser_use_version()
588
+
589
+ # Determine source
590
+ try:
591
+ package_root = Path(__file__).parent.parent.parent
592
+ repo_files = ['.git', 'README.md', 'docs', 'examples']
593
+ if all(Path(package_root / file).exists() for file in repo_files):
594
+ source = 'git'
595
+ else:
596
+ source = 'pip'
597
+ except Exception as e:
598
+ self.logger.debug(f'Error determining source: {e}')
599
+ source = 'unknown'
600
+
601
+ if source_override is not None:
602
+ source = source_override
603
+ # self.logger.debug(f'Version: {version}, Source: {source}') # moved later to _log_agent_run so that people are more likely to include it in copy-pasted support ticket logs
604
+ self.version = version
605
+ self.source = source
606
+
607
+ def _setup_action_models(self) -> None:
608
+ """Setup dynamic action models from tools registry"""
609
+ # Initially only include actions with no filters
610
+ self.ActionModel = self.tools.registry.create_action_model()
611
+ # Create output model with the dynamic actions
612
+ if self.settings.flash_mode:
613
+ self.AgentOutput = AgentOutput.type_with_custom_actions_flash_mode(self.ActionModel)
614
+ elif self.settings.use_thinking:
615
+ self.AgentOutput = AgentOutput.type_with_custom_actions(self.ActionModel)
616
+ else:
617
+ self.AgentOutput = AgentOutput.type_with_custom_actions_no_thinking(self.ActionModel)
618
+
619
+ # used to force the done action when max_steps is reached
620
+ self.DoneActionModel = self.tools.registry.create_action_model(include_actions=['done'])
621
+ if self.settings.flash_mode:
622
+ self.DoneAgentOutput = AgentOutput.type_with_custom_actions_flash_mode(self.DoneActionModel)
623
+ elif self.settings.use_thinking:
624
+ self.DoneAgentOutput = AgentOutput.type_with_custom_actions(self.DoneActionModel)
625
+ else:
626
+ self.DoneAgentOutput = AgentOutput.type_with_custom_actions_no_thinking(self.DoneActionModel)
627
+
628
+ def add_new_task(self, new_task: str) -> None:
629
+ """Add a new task to the agent, keeping the same task_id as tasks are continuous"""
630
+ # Simply delegate to message manager - no need for new task_id or events
631
+ # The task continues with new instructions, it doesn't end and start a new one
632
+ self.task = new_task
633
+ self._message_manager.add_new_task(new_task)
634
+ # Mark as follow-up task and recreate eventbus (gets shut down after each run)
635
+ self.state.follow_up_task = True
636
+ # Reset control flags so agent can continue
637
+ self.state.stopped = False
638
+ self.state.paused = False
639
+ agent_id_suffix = str(self.id)[-4:].replace('-', '_')
640
+ if agent_id_suffix and agent_id_suffix[0].isdigit():
641
+ agent_id_suffix = 'a' + agent_id_suffix
642
+ self.eventbus = EventBus(name=f'Agent_{agent_id_suffix}')
643
+
644
+ async def _check_stop_or_pause(self) -> None:
645
+ """Check if the agent should stop or pause, and handle accordingly."""
646
+
647
+ # Check new should_stop_callback - sets stopped state cleanly without raising
648
+ if self.register_should_stop_callback:
649
+ if await self.register_should_stop_callback():
650
+ self.logger.info('External callback requested stop')
651
+ self.state.stopped = True
652
+ raise InterruptedError
653
+
654
+ if self.register_external_agent_status_raise_error_callback:
655
+ if await self.register_external_agent_status_raise_error_callback():
656
+ raise InterruptedError
657
+
658
+ if self.state.stopped:
659
+ raise InterruptedError
660
+
661
+ if self.state.paused:
662
+ raise InterruptedError
663
+
664
+ @observe(name='agent.step', ignore_output=True, ignore_input=True)
665
+ @time_execution_async('--step')
666
+ async def step(self, step_info: AgentStepInfo | None = None) -> None:
667
+ """Execute one step of the task"""
668
+ # Initialize timing first, before any exceptions can occur
669
+
670
+ self.step_start_time = time.time()
671
+
672
+ browser_state_summary = None
673
+
674
+ try:
675
+ # Phase 1: Prepare context and timing
676
+ browser_state_summary = await self._prepare_context(step_info)
677
+
678
+ # Phase 2: Get model output and execute actions
679
+ await self._get_next_action(browser_state_summary)
680
+ await self._execute_actions()
681
+
682
+ # Phase 3: Post-processing
683
+ await self._post_process()
684
+
685
+ except Exception as e:
686
+ # Handle ALL exceptions in one place
687
+ await self._handle_step_error(e)
688
+
689
+ finally:
690
+ await self._finalize(browser_state_summary)
691
+
692
+ async def _prepare_context(self, step_info: AgentStepInfo | None = None) -> BrowserStateSummary:
693
+ """Prepare the context for the step: browser state, action models, page actions"""
694
+ # step_start_time is now set in step() method
695
+
696
+ assert self.browser_session is not None, 'BrowserSession is not set up'
697
+
698
+ self.logger.debug(f'🌐 Step {self.state.n_steps}: Getting browser state...')
699
+ # Always take screenshots for all steps
700
+ self.logger.debug('📸 Requesting browser state with include_screenshot=True')
701
+ browser_state_summary = await self.browser_session.get_browser_state_summary(
702
+ include_screenshot=True, # always capture even if use_vision=False so that cloud sync is useful (it's fast now anyway)
703
+ include_recent_events=self.include_recent_events,
704
+ )
705
+ if browser_state_summary.screenshot:
706
+ self.logger.debug(f'📸 Got browser state WITH screenshot, length: {len(browser_state_summary.screenshot)}')
707
+ else:
708
+ self.logger.debug('📸 Got browser state WITHOUT screenshot')
709
+
710
+ # Check for new downloads after getting browser state (catches PDF auto-downloads and previous step downloads)
711
+ await self._check_and_update_downloads(f'Step {self.state.n_steps}: after getting browser state')
712
+
713
+ self._log_step_context(browser_state_summary)
714
+ await self._check_stop_or_pause()
715
+
716
+ # Update action models with page-specific actions
717
+ self.logger.debug(f'📝 Step {self.state.n_steps}: Updating action models...')
718
+ await self._update_action_models_for_page(browser_state_summary.url)
719
+
720
+ # Get page-specific filtered actions
721
+ page_filtered_actions = self.tools.registry.get_prompt_description(browser_state_summary.url)
722
+
723
+ # Page-specific actions will be included directly in the browser_state message
724
+ self.logger.debug(f'💬 Step {self.state.n_steps}: Creating state messages for context...')
725
+
726
+ self._message_manager.create_state_messages(
727
+ browser_state_summary=browser_state_summary,
728
+ model_output=self.state.last_model_output,
729
+ result=self.state.last_result,
730
+ step_info=step_info,
731
+ use_vision=self.settings.use_vision,
732
+ page_filtered_actions=page_filtered_actions if page_filtered_actions else None,
733
+ sensitive_data=self.sensitive_data,
734
+ available_file_paths=self.available_file_paths, # Always pass current available_file_paths
735
+ )
736
+
737
+ await self._force_done_after_last_step(step_info)
738
+ await self._force_done_after_failure()
739
+ return browser_state_summary
740
+
741
+ @observe_debug(ignore_input=True, name='get_next_action')
742
+ async def _get_next_action(self, browser_state_summary: BrowserStateSummary) -> None:
743
+ """Execute LLM interaction with retry logic and handle callbacks"""
744
+ input_messages = self._message_manager.get_messages()
745
+ self.logger.debug(
746
+ f'🤖 Step {self.state.n_steps}: Calling LLM with {len(input_messages)} messages (model: {self.llm.model})...'
747
+ )
748
+
749
+ try:
750
+ model_output = await asyncio.wait_for(
751
+ self._get_model_output_with_retry(input_messages), timeout=self.settings.llm_timeout
752
+ )
753
+ except TimeoutError:
754
+
755
+ @observe(name='_llm_call_timed_out_with_input')
756
+ async def _log_model_input_to_lmnr(input_messages: list[BaseMessage]) -> None:
757
+ """Log the model input"""
758
+ pass
759
+
760
+ await _log_model_input_to_lmnr(input_messages)
761
+
762
+ raise TimeoutError(
763
+ f'LLM call timed out after {self.settings.llm_timeout} seconds. Keep your thinking and output short.'
764
+ )
765
+
766
+ self.state.last_model_output = model_output
767
+
768
+ # Check again for paused/stopped state after getting model output
769
+ await self._check_stop_or_pause()
770
+
771
+ # Handle callbacks and conversation saving
772
+ await self._handle_post_llm_processing(browser_state_summary, input_messages)
773
+
774
+ # check again if Ctrl+C was pressed before we commit the output to history
775
+ await self._check_stop_or_pause()
776
+
777
+ async def _execute_actions(self) -> None:
778
+ """Execute the actions from model output"""
779
+ if self.state.last_model_output is None:
780
+ raise ValueError('No model output to execute actions from')
781
+
782
+ result = await self.multi_act(self.state.last_model_output.action)
783
+ self.state.last_result = result
784
+
785
+ async def _post_process(self) -> None:
786
+ """Handle post-action processing like download tracking and result logging"""
787
+ assert self.browser_session is not None, 'BrowserSession is not set up'
788
+
789
+ # Check for new downloads after executing actions
790
+ await self._check_and_update_downloads('after executing actions')
791
+
792
+ # check for action errors and len more than 1
793
+ if self.state.last_result and len(self.state.last_result) == 1 and self.state.last_result[-1].error:
794
+ self.state.consecutive_failures += 1
795
+ self.logger.debug(f'🔄 Step {self.state.n_steps}: Consecutive failures: {self.state.consecutive_failures}')
796
+ return
797
+
798
+ if self.state.consecutive_failures > 0:
799
+ self.state.consecutive_failures = 0
800
+ self.logger.debug(f'🔄 Step {self.state.n_steps}: Consecutive failures reset to: {self.state.consecutive_failures}')
801
+
802
+ # Log completion results
803
+ if self.state.last_result and len(self.state.last_result) > 0 and self.state.last_result[-1].is_done:
804
+ success = self.state.last_result[-1].success
805
+ if success:
806
+ # Green color for success
807
+ self.logger.info(f'\n📄 \033[32m Final Result:\033[0m \n{self.state.last_result[-1].extracted_content}\n\n')
808
+ else:
809
+ # Red color for failure
810
+ self.logger.info(f'\n📄 \033[31m Final Result:\033[0m \n{self.state.last_result[-1].extracted_content}\n\n')
811
+ if self.state.last_result[-1].attachments:
812
+ total_attachments = len(self.state.last_result[-1].attachments)
813
+ for i, file_path in enumerate(self.state.last_result[-1].attachments):
814
+ self.logger.info(f'👉 Attachment {i + 1 if total_attachments > 1 else ""}: {file_path}')
815
+
816
+ async def _handle_step_error(self, error: Exception) -> None:
817
+ """Handle all types of errors that can occur during a step"""
818
+
819
+ # Handle InterruptedError specially
820
+ if isinstance(error, InterruptedError):
821
+ error_msg = 'The agent was interrupted mid-step' + (f' - {str(error)}' if str(error) else '')
822
+ self.logger.error(f'{error_msg}')
823
+ return
824
+
825
+ # Handle all other exceptions
826
+ include_trace = self.logger.isEnabledFor(logging.DEBUG)
827
+ error_msg = AgentError.format_error(error, include_trace=include_trace)
828
+ prefix = f'❌ Result failed {self.state.consecutive_failures + 1}/{self.settings.max_failures + int(self.settings.final_response_after_failure)} times:\n '
829
+ self.state.consecutive_failures += 1
830
+
831
+ if 'Could not parse response' in error_msg or 'tool_use_failed' in error_msg:
832
+ # give model a hint how output should look like
833
+ logger.error(f'Model: {self.llm.model} failed')
834
+ logger.error(f'{prefix}{error_msg}')
835
+ else:
836
+ self.logger.error(f'{prefix}{error_msg}')
837
+
838
+ self.state.last_result = [ActionResult(error=error_msg)]
839
+ return None
840
+
841
+ async def _finalize(self, browser_state_summary: BrowserStateSummary | None) -> None:
842
+ """Finalize the step with history, logging, and events"""
843
+ step_end_time = time.time()
844
+ if not self.state.last_result:
845
+ return
846
+
847
+ if browser_state_summary:
848
+ metadata = StepMetadata(
849
+ step_number=self.state.n_steps,
850
+ step_start_time=self.step_start_time,
851
+ step_end_time=step_end_time,
852
+ )
853
+
854
+ # Use _make_history_item like main branch
855
+ await self._make_history_item(
856
+ self.state.last_model_output,
857
+ browser_state_summary,
858
+ self.state.last_result,
859
+ metadata,
860
+ state_message=self._message_manager.last_state_message_text,
861
+ )
862
+
863
+ # Log step completion summary
864
+ self._log_step_completion_summary(self.step_start_time, self.state.last_result)
865
+
866
+ # Save file system state after step completion
867
+ self.save_file_system_state()
868
+
869
+ # Emit both step created and executed events
870
+ if browser_state_summary and self.state.last_model_output:
871
+ # Extract key step data for the event
872
+ actions_data = []
873
+ if self.state.last_model_output.action:
874
+ for action in self.state.last_model_output.action:
875
+ action_dict = action.model_dump() if hasattr(action, 'model_dump') else {}
876
+ actions_data.append(action_dict)
877
+
878
+ # Emit CreateAgentStepEvent
879
+ step_event = CreateAgentStepEvent.from_agent_step(
880
+ self,
881
+ self.state.last_model_output,
882
+ self.state.last_result,
883
+ actions_data,
884
+ browser_state_summary,
885
+ )
886
+ self.eventbus.dispatch(step_event)
887
+
888
+ # Increment step counter after step is fully completed
889
+ self.state.n_steps += 1
890
+
891
+ async def _force_done_after_last_step(self, step_info: AgentStepInfo | None = None) -> None:
892
+ """Handle special processing for the last step"""
893
+ if step_info and step_info.is_last_step():
894
+ # Add last step warning if needed
895
+ msg = 'You reached max_steps - this is your last step. Your only tool available is the "done" tool. No other tool is available. All other tools which you see in history or examples are not available.'
896
+ msg += '\nIf the task is not yet fully finished as requested by the user, set success in "done" to false! E.g. if not all steps are fully completed. Else success to true.'
897
+ msg += '\nInclude everything you found out for the ultimate task in the done text.'
898
+ self.logger.debug('Last step finishing up')
899
+ self._message_manager._add_context_message(UserMessage(content=msg))
900
+ self.AgentOutput = self.DoneAgentOutput
901
+
902
+ async def _force_done_after_failure(self) -> None:
903
+ """Force done after failure"""
904
+ # Create recovery message
905
+ if self.state.consecutive_failures >= self.settings.max_failures and self.settings.final_response_after_failure:
906
+ msg = f'You failed {self.settings.max_failures} times. Therefore we terminate the agent.'
907
+ msg += '\nYour only tool available is the "done" tool. No other tool is available. All other tools which you see in history or examples are not available.'
908
+ msg += '\nIf the task is not yet fully finished as requested by the user, set success in "done" to false! E.g. if not all steps are fully completed. Else success to true.'
909
+ msg += '\nInclude everything you found out for the ultimate task in the done text.'
910
+
911
+ self.logger.debug('Force done action, because we reached max_failures.')
912
+ self._message_manager._add_context_message(UserMessage(content=msg))
913
+ self.AgentOutput = self.DoneAgentOutput
914
+
915
+ @observe(ignore_input=True, ignore_output=False)
916
+ async def _judge_trace(self) -> JudgementResult | None:
917
+ """Judge the trace of the agent"""
918
+ task = self.task
919
+ final_result = self.history.final_result() or ''
920
+ agent_steps = self.history.agent_steps()
921
+ screenshot_paths = [p for p in self.history.screenshot_paths() if p is not None]
922
+
923
+ # Construct input messages for judge evaluation
924
+ input_messages = construct_judge_messages(
925
+ task=task,
926
+ final_result=final_result,
927
+ agent_steps=agent_steps,
928
+ screenshot_paths=screenshot_paths,
929
+ max_images=10,
930
+ )
931
+
932
+ # Call LLM with JudgementResult as output format
933
+ kwargs: dict = {'output_format': JudgementResult}
934
+
935
+ # Only pass request_type for ChatBrowserUse (other providers don't support it)
936
+ if self.judge_llm.provider == 'browser-use':
937
+ kwargs['request_type'] = 'judge'
938
+
939
+ try:
940
+ response = await self.judge_llm.ainvoke(input_messages, **kwargs)
941
+ judgement: JudgementResult = response.completion # type: ignore[assignment]
942
+ return judgement
943
+ except Exception as e:
944
+ self.logger.error(f'Judge trace failed: {e}')
945
+ # Return a default judgement on failure
946
+ return None
947
+
948
+ async def _judge_and_log(self) -> None:
949
+ """Run judge evaluation and log the verdict"""
950
+ judgement = await self._judge_trace()
951
+
952
+ # Attach judgement to last action result
953
+ if self.history.history[-1].result[-1].is_done:
954
+ last_result = self.history.history[-1].result[-1]
955
+ last_result.judgement = judgement
956
+
957
+ # Get self-reported success
958
+ self_reported_success = last_result.success
959
+
960
+ # Log the verdict based on self-reported success and judge verdict
961
+ if judgement:
962
+ # If both self-reported and judge agree on success, don't log
963
+ if self_reported_success is True and judgement.verdict is True:
964
+ return
965
+
966
+ judge_log = '\n'
967
+ # If agent reported success but judge thinks it failed, show warning
968
+ if self_reported_success is True and judgement.verdict is False:
969
+ judge_log += '⚠️ \033[33mAgent reported success but judge thinks task failed\033[0m\n'
970
+
971
+ # Otherwise, show full judge result
972
+ verdict_color = '\033[32m' if judgement.verdict else '\033[31m'
973
+ verdict_text = '✅ PASS' if judgement.verdict else '❌ FAIL'
974
+ judge_log += f'⚖️ {verdict_color}Judge Verdict: {verdict_text}\033[0m\n'
975
+ if judgement.failure_reason:
976
+ judge_log += f' Failure: {judgement.failure_reason}\n'
977
+ judge_log += f' {judgement.reasoning}\n'
978
+ self.logger.info(judge_log)
979
+
980
+ async def _get_model_output_with_retry(self, input_messages: list[BaseMessage]) -> AgentOutput:
981
+ """Get model output with retry logic for empty actions"""
982
+ model_output = await self.get_model_output(input_messages)
983
+ self.logger.debug(
984
+ f'✅ Step {self.state.n_steps}: Got LLM response with {len(model_output.action) if model_output.action else 0} actions'
985
+ )
986
+
987
+ if (
988
+ not model_output.action
989
+ or not isinstance(model_output.action, list)
990
+ or all(action.model_dump() == {} for action in model_output.action)
991
+ ):
992
+ self.logger.warning('Model returned empty action. Retrying...')
993
+
994
+ clarification_message = UserMessage(
995
+ content='You forgot to return an action. Please respond with a valid JSON action according to the expected schema with your assessment and next actions.'
996
+ )
997
+
998
+ retry_messages = input_messages + [clarification_message]
999
+ model_output = await self.get_model_output(retry_messages)
1000
+
1001
+ if not model_output.action or all(action.model_dump() == {} for action in model_output.action):
1002
+ self.logger.warning('Model still returned empty after retry. Inserting safe noop action.')
1003
+ action_instance = self.ActionModel()
1004
+ setattr(
1005
+ action_instance,
1006
+ 'done',
1007
+ {
1008
+ 'success': False,
1009
+ 'text': 'No next action returned by LLM!',
1010
+ },
1011
+ )
1012
+ model_output.action = [action_instance]
1013
+
1014
+ return model_output
1015
+
1016
+ async def _handle_post_llm_processing(
1017
+ self,
1018
+ browser_state_summary: BrowserStateSummary,
1019
+ input_messages: list[BaseMessage],
1020
+ ) -> None:
1021
+ """Handle callbacks and conversation saving after LLM interaction"""
1022
+ if self.register_new_step_callback and self.state.last_model_output:
1023
+ if inspect.iscoroutinefunction(self.register_new_step_callback):
1024
+ await self.register_new_step_callback(
1025
+ browser_state_summary,
1026
+ self.state.last_model_output,
1027
+ self.state.n_steps,
1028
+ )
1029
+ else:
1030
+ self.register_new_step_callback(
1031
+ browser_state_summary,
1032
+ self.state.last_model_output,
1033
+ self.state.n_steps,
1034
+ )
1035
+
1036
+ if self.settings.save_conversation_path and self.state.last_model_output:
1037
+ # Treat save_conversation_path as a directory (consistent with other recording paths)
1038
+ conversation_dir = Path(self.settings.save_conversation_path)
1039
+ conversation_filename = f'conversation_{self.id}_{self.state.n_steps}.txt'
1040
+ target = conversation_dir / conversation_filename
1041
+ await save_conversation(
1042
+ input_messages,
1043
+ self.state.last_model_output,
1044
+ target,
1045
+ self.settings.save_conversation_path_encoding,
1046
+ )
1047
+
1048
+ async def _make_history_item(
1049
+ self,
1050
+ model_output: AgentOutput | None,
1051
+ browser_state_summary: BrowserStateSummary,
1052
+ result: list[ActionResult],
1053
+ metadata: StepMetadata | None = None,
1054
+ state_message: str | None = None,
1055
+ ) -> None:
1056
+ """Create and store history item"""
1057
+
1058
+ if model_output:
1059
+ interacted_elements = AgentHistory.get_interacted_element(model_output, browser_state_summary.dom_state.selector_map)
1060
+ else:
1061
+ interacted_elements = [None]
1062
+
1063
+ # Store screenshot and get path
1064
+ screenshot_path = None
1065
+ if browser_state_summary.screenshot:
1066
+ self.logger.debug(
1067
+ f'📸 Storing screenshot for step {self.state.n_steps}, screenshot length: {len(browser_state_summary.screenshot)}'
1068
+ )
1069
+ screenshot_path = await self.screenshot_service.store_screenshot(browser_state_summary.screenshot, self.state.n_steps)
1070
+ self.logger.debug(f'📸 Screenshot stored at: {screenshot_path}')
1071
+ else:
1072
+ self.logger.debug(f'📸 No screenshot in browser_state_summary for step {self.state.n_steps}')
1073
+
1074
+ state_history = BrowserStateHistory(
1075
+ url=browser_state_summary.url,
1076
+ title=browser_state_summary.title,
1077
+ tabs=browser_state_summary.tabs,
1078
+ interacted_element=interacted_elements,
1079
+ screenshot_path=screenshot_path,
1080
+ )
1081
+
1082
+ history_item = AgentHistory(
1083
+ model_output=model_output,
1084
+ result=result,
1085
+ state=state_history,
1086
+ metadata=metadata,
1087
+ state_message=state_message,
1088
+ )
1089
+
1090
+ self.history.add_item(history_item)
1091
+
1092
+ def _remove_think_tags(self, text: str) -> str:
1093
+ THINK_TAGS = re.compile(r'<think>.*?</think>', re.DOTALL)
1094
+ STRAY_CLOSE_TAG = re.compile(r'.*?</think>', re.DOTALL)
1095
+ # Step 1: Remove well-formed <think>...</think>
1096
+ text = re.sub(THINK_TAGS, '', text)
1097
+ # Step 2: If there's an unmatched closing tag </think>,
1098
+ # remove everything up to and including that.
1099
+ text = re.sub(STRAY_CLOSE_TAG, '', text)
1100
+ return text.strip()
1101
+
1102
+ # region - URL replacement
1103
+ def _replace_urls_in_text(self, text: str) -> tuple[str, dict[str, str]]:
1104
+ """Replace URLs in a text string"""
1105
+
1106
+ replaced_urls: dict[str, str] = {}
1107
+
1108
+ def replace_url(match: re.Match) -> str:
1109
+ """Url can only have 1 query and 1 fragment"""
1110
+ import hashlib
1111
+
1112
+ original_url = match.group(0)
1113
+
1114
+ # Find where the query/fragment starts
1115
+ query_start = original_url.find('?')
1116
+ fragment_start = original_url.find('#')
1117
+
1118
+ # Find the earliest position of query or fragment
1119
+ after_path_start = len(original_url) # Default: no query/fragment
1120
+ if query_start != -1:
1121
+ after_path_start = min(after_path_start, query_start)
1122
+ if fragment_start != -1:
1123
+ after_path_start = min(after_path_start, fragment_start)
1124
+
1125
+ # Split URL into base (up to path) and after_path (query + fragment)
1126
+ base_url = original_url[:after_path_start]
1127
+ after_path = original_url[after_path_start:]
1128
+
1129
+ # If after_path is within the limit, don't shorten
1130
+ if len(after_path) <= self._url_shortening_limit:
1131
+ return original_url
1132
+
1133
+ # If after_path is too long, truncate and add hash
1134
+ if after_path:
1135
+ truncated_after_path = after_path[: self._url_shortening_limit]
1136
+ # Create a short hash of the full after_path content
1137
+ hash_obj = hashlib.md5(after_path.encode('utf-8'))
1138
+ short_hash = hash_obj.hexdigest()[:7]
1139
+ # Create shortened URL
1140
+ shortened = f'{base_url}{truncated_after_path}...{short_hash}'
1141
+ # Only use shortened URL if it's actually shorter than the original
1142
+ if len(shortened) < len(original_url):
1143
+ replaced_urls[shortened] = original_url
1144
+ return shortened
1145
+
1146
+ return original_url
1147
+
1148
+ return URL_PATTERN.sub(replace_url, text), replaced_urls
1149
+
1150
+ def _process_messsages_and_replace_long_urls_shorter_ones(self, input_messages: list[BaseMessage]) -> dict[str, str]:
1151
+ """Replace long URLs with shorter ones
1152
+ ? @dev edits input_messages in place
1153
+
1154
+ returns:
1155
+ tuple[filtered_input_messages, urls we replaced {shorter_url: original_url}]
1156
+ """
1157
+ from browser_use.llm.messages import AssistantMessage, UserMessage
1158
+
1159
+ urls_replaced: dict[str, str] = {}
1160
+
1161
+ # Process each message, in place
1162
+ for message in input_messages:
1163
+ # no need to process SystemMessage, we have control over that anyway
1164
+ if isinstance(message, (UserMessage, AssistantMessage)):
1165
+ if isinstance(message.content, str):
1166
+ # Simple string content
1167
+ message.content, replaced_urls = self._replace_urls_in_text(message.content)
1168
+ urls_replaced.update(replaced_urls)
1169
+
1170
+ elif isinstance(message.content, list):
1171
+ # List of content parts
1172
+ for part in message.content:
1173
+ if isinstance(part, ContentPartTextParam):
1174
+ part.text, replaced_urls = self._replace_urls_in_text(part.text)
1175
+ urls_replaced.update(replaced_urls)
1176
+
1177
+ return urls_replaced
1178
+
1179
+ @staticmethod
1180
+ def _recursive_process_all_strings_inside_pydantic_model(model: BaseModel, url_replacements: dict[str, str]) -> None:
1181
+ """Recursively process all strings inside a Pydantic model, replacing shortened URLs with originals in place."""
1182
+ for field_name, field_value in model.__dict__.items():
1183
+ if isinstance(field_value, str):
1184
+ # Replace shortened URLs with original URLs in string
1185
+ processed_string = Agent._replace_shortened_urls_in_string(field_value, url_replacements)
1186
+ setattr(model, field_name, processed_string)
1187
+ elif isinstance(field_value, BaseModel):
1188
+ # Recursively process nested Pydantic models
1189
+ Agent._recursive_process_all_strings_inside_pydantic_model(field_value, url_replacements)
1190
+ elif isinstance(field_value, dict):
1191
+ # Process dictionary values in place
1192
+ Agent._recursive_process_dict(field_value, url_replacements)
1193
+ elif isinstance(field_value, (list, tuple)):
1194
+ processed_value = Agent._recursive_process_list_or_tuple(field_value, url_replacements)
1195
+ setattr(model, field_name, processed_value)
1196
+
1197
+ @staticmethod
1198
+ def _recursive_process_dict(dictionary: dict, url_replacements: dict[str, str]) -> None:
1199
+ """Helper method to process dictionaries."""
1200
+ for k, v in dictionary.items():
1201
+ if isinstance(v, str):
1202
+ dictionary[k] = Agent._replace_shortened_urls_in_string(v, url_replacements)
1203
+ elif isinstance(v, BaseModel):
1204
+ Agent._recursive_process_all_strings_inside_pydantic_model(v, url_replacements)
1205
+ elif isinstance(v, dict):
1206
+ Agent._recursive_process_dict(v, url_replacements)
1207
+ elif isinstance(v, (list, tuple)):
1208
+ dictionary[k] = Agent._recursive_process_list_or_tuple(v, url_replacements)
1209
+
1210
+ @staticmethod
1211
+ def _recursive_process_list_or_tuple(container: list | tuple, url_replacements: dict[str, str]) -> list | tuple:
1212
+ """Helper method to process lists and tuples."""
1213
+ if isinstance(container, tuple):
1214
+ # For tuples, create a new tuple with processed items
1215
+ processed_items = []
1216
+ for item in container:
1217
+ if isinstance(item, str):
1218
+ processed_items.append(Agent._replace_shortened_urls_in_string(item, url_replacements))
1219
+ elif isinstance(item, BaseModel):
1220
+ Agent._recursive_process_all_strings_inside_pydantic_model(item, url_replacements)
1221
+ processed_items.append(item)
1222
+ elif isinstance(item, dict):
1223
+ Agent._recursive_process_dict(item, url_replacements)
1224
+ processed_items.append(item)
1225
+ elif isinstance(item, (list, tuple)):
1226
+ processed_items.append(Agent._recursive_process_list_or_tuple(item, url_replacements))
1227
+ else:
1228
+ processed_items.append(item)
1229
+ return tuple(processed_items)
1230
+ else:
1231
+ # For lists, modify in place
1232
+ for i, item in enumerate(container):
1233
+ if isinstance(item, str):
1234
+ container[i] = Agent._replace_shortened_urls_in_string(item, url_replacements)
1235
+ elif isinstance(item, BaseModel):
1236
+ Agent._recursive_process_all_strings_inside_pydantic_model(item, url_replacements)
1237
+ elif isinstance(item, dict):
1238
+ Agent._recursive_process_dict(item, url_replacements)
1239
+ elif isinstance(item, (list, tuple)):
1240
+ container[i] = Agent._recursive_process_list_or_tuple(item, url_replacements)
1241
+ return container
1242
+
1243
+ @staticmethod
1244
+ def _replace_shortened_urls_in_string(text: str, url_replacements: dict[str, str]) -> str:
1245
+ """Replace all shortened URLs in a string with their original URLs."""
1246
+ result = text
1247
+ for shortened_url, original_url in url_replacements.items():
1248
+ result = result.replace(shortened_url, original_url)
1249
+ return result
1250
+
1251
+ # endregion - URL replacement
1252
+
1253
+ @time_execution_async('--get_next_action')
1254
+ @observe_debug(ignore_input=True, ignore_output=True, name='get_model_output')
1255
+ async def get_model_output(self, input_messages: list[BaseMessage]) -> AgentOutput:
1256
+ """Get next action from LLM based on current state"""
1257
+
1258
+ urls_replaced = self._process_messsages_and_replace_long_urls_shorter_ones(input_messages)
1259
+
1260
+ # Build kwargs for ainvoke
1261
+ # Note: ChatBrowserUse will automatically generate action descriptions from output_format schema
1262
+ kwargs: dict = {'output_format': self.AgentOutput}
1263
+
1264
+ try:
1265
+ response = await self.llm.ainvoke(input_messages, **kwargs)
1266
+ parsed: AgentOutput = response.completion # type: ignore[assignment]
1267
+
1268
+ # Replace any shortened URLs in the LLM response back to original URLs
1269
+ if urls_replaced:
1270
+ self._recursive_process_all_strings_inside_pydantic_model(parsed, urls_replaced)
1271
+
1272
+ # cut the number of actions to max_actions_per_step if needed
1273
+ if len(parsed.action) > self.settings.max_actions_per_step:
1274
+ parsed.action = parsed.action[: self.settings.max_actions_per_step]
1275
+
1276
+ if not (hasattr(self.state, 'paused') and (self.state.paused or self.state.stopped)):
1277
+ log_response(parsed, self.tools.registry.registry, self.logger)
1278
+
1279
+ self._log_next_action_summary(parsed)
1280
+ return parsed
1281
+ except ValidationError:
1282
+ # Just re-raise - Pydantic's validation errors are already descriptive
1283
+ raise
1284
+
1285
+ async def _log_agent_run(self) -> None:
1286
+ """Log the agent run"""
1287
+ # Blue color for task
1288
+ self.logger.info(f'\033[34m🎯 Task: {self.task}\033[0m')
1289
+
1290
+ self.logger.debug(f'🤖 Browser-Use Library Version {self.version} ({self.source})')
1291
+
1292
+ # Check for latest version and log upgrade message if needed
1293
+ latest_version = await check_latest_browser_use_version()
1294
+ if latest_version and latest_version != self.version:
1295
+ self.logger.info(
1296
+ f'📦 Newer version available: {latest_version} (current: {self.version}). Upgrade with: uv add browser-use@{latest_version}'
1297
+ )
1298
+
1299
+ def _log_first_step_startup(self) -> None:
1300
+ """Log startup message only on the first step"""
1301
+ if len(self.history.history) == 0:
1302
+ self.logger.info(
1303
+ f'Starting a browser-use agent with version {self.version}, with provider={self.llm.provider} and model={self.llm.model}'
1304
+ )
1305
+
1306
+ def _log_step_context(self, browser_state_summary: BrowserStateSummary) -> None:
1307
+ """Log step context information"""
1308
+ url = browser_state_summary.url if browser_state_summary else ''
1309
+ url_short = url[:50] + '...' if len(url) > 50 else url
1310
+ interactive_count = len(browser_state_summary.dom_state.selector_map) if browser_state_summary else 0
1311
+ self.logger.info('\n')
1312
+ self.logger.info(f'📍 Step {self.state.n_steps}:')
1313
+ self.logger.debug(f'Evaluating page with {interactive_count} interactive elements on: {url_short}')
1314
+
1315
+ def _log_next_action_summary(self, parsed: 'AgentOutput') -> None:
1316
+ """Log a comprehensive summary of the next action(s)"""
1317
+ if not (self.logger.isEnabledFor(logging.DEBUG) and parsed.action):
1318
+ return
1319
+
1320
+ action_count = len(parsed.action)
1321
+
1322
+ # Collect action details
1323
+ action_details = []
1324
+ for i, action in enumerate(parsed.action):
1325
+ action_data = action.model_dump(exclude_unset=True)
1326
+ action_name = next(iter(action_data.keys())) if action_data else 'unknown'
1327
+ action_params = action_data.get(action_name, {}) if action_data else {}
1328
+
1329
+ # Format key parameters concisely
1330
+ param_summary = []
1331
+ if isinstance(action_params, dict):
1332
+ for key, value in action_params.items():
1333
+ if key == 'index':
1334
+ param_summary.append(f'#{value}')
1335
+ elif key == 'text' and isinstance(value, str):
1336
+ text_preview = value[:30] + '...' if len(value) > 30 else value
1337
+ param_summary.append(f'text="{text_preview}"')
1338
+ elif key == 'url':
1339
+ param_summary.append(f'url="{value}"')
1340
+ elif key == 'success':
1341
+ param_summary.append(f'success={value}')
1342
+ elif isinstance(value, (str, int, bool)):
1343
+ val_str = str(value)[:30] + '...' if len(str(value)) > 30 else str(value)
1344
+ param_summary.append(f'{key}={val_str}')
1345
+
1346
+ param_str = f'({", ".join(param_summary)})' if param_summary else ''
1347
+ action_details.append(f'{action_name}{param_str}')
1348
+
1349
+ def _log_step_completion_summary(self, step_start_time: float, result: list[ActionResult]) -> None:
1350
+ """Log step completion summary with action count, timing, and success/failure stats"""
1351
+ if not result:
1352
+ return
1353
+
1354
+ step_duration = time.time() - step_start_time
1355
+ action_count = len(result)
1356
+
1357
+ # Count success and failures
1358
+ success_count = sum(1 for r in result if not r.error)
1359
+ failure_count = action_count - success_count
1360
+
1361
+ # Format success/failure indicators
1362
+ success_indicator = f'✅ {success_count}' if success_count > 0 else ''
1363
+ failure_indicator = f'❌ {failure_count}' if failure_count > 0 else ''
1364
+ status_parts = [part for part in [success_indicator, failure_indicator] if part]
1365
+ status_str = ' | '.join(status_parts) if status_parts else '✅ 0'
1366
+
1367
+ self.logger.debug(
1368
+ f'📍 Step {self.state.n_steps}: Ran {action_count} action{"" if action_count == 1 else "s"} in {step_duration:.2f}s: {status_str}'
1369
+ )
1370
+
1371
+ def _log_final_outcome_messages(self) -> None:
1372
+ """Log helpful messages to user based on agent run outcome"""
1373
+ # Check if agent failed
1374
+ is_successful = self.history.is_successful()
1375
+
1376
+ if is_successful is False or is_successful is None:
1377
+ # Get final result to check for specific failure reasons
1378
+ final_result = self.history.final_result()
1379
+ final_result_str = str(final_result).lower() if final_result else ''
1380
+
1381
+ # Check for captcha/cloudflare related failures
1382
+ captcha_keywords = ['captcha', 'cloudflare', 'recaptcha', 'challenge', 'bot detection', 'access denied']
1383
+ has_captcha_issue = any(keyword in final_result_str for keyword in captcha_keywords)
1384
+
1385
+ if has_captcha_issue:
1386
+ # Suggest use_cloud=True for captcha/cloudflare issues
1387
+ task_preview = self.task[:10] if len(self.task) > 10 else self.task
1388
+ self.logger.info('')
1389
+ self.logger.info('Failed because of CAPTCHA? For better browser stealth, try:')
1390
+ self.logger.info(f' agent = Agent(task="{task_preview}...", browser=Browser(use_cloud=True))')
1391
+
1392
+ # General failure message
1393
+ self.logger.info('')
1394
+ self.logger.info('Did the Agent not work as expected? Let us fix this!')
1395
+ self.logger.info(' Open a short issue on GitHub: https://github.com/browser-use/browser-use/issues')
1396
+
1397
+ def _log_agent_event(self, max_steps: int, agent_run_error: str | None = None) -> None:
1398
+ """Sent the agent event for this run to telemetry"""
1399
+
1400
+ token_summary = self.token_cost_service.get_usage_tokens_for_model(self.llm.model)
1401
+
1402
+ # Prepare action_history data correctly
1403
+ action_history_data = []
1404
+ for item in self.history.history:
1405
+ if item.model_output and item.model_output.action:
1406
+ # Convert each ActionModel in the step to its dictionary representation
1407
+ step_actions = [
1408
+ action.model_dump(exclude_unset=True)
1409
+ for action in item.model_output.action
1410
+ if action # Ensure action is not None if list allows it
1411
+ ]
1412
+ action_history_data.append(step_actions)
1413
+ else:
1414
+ # Append None or [] if a step had no actions or no model output
1415
+ action_history_data.append(None)
1416
+
1417
+ final_res = self.history.final_result()
1418
+ final_result_str = json.dumps(final_res) if final_res is not None else None
1419
+
1420
+ # Extract judgement data if available
1421
+ judgement_data = self.history.judgement()
1422
+ judge_verdict = judgement_data.get('verdict') if judgement_data else None
1423
+ judge_reasoning = judgement_data.get('reasoning') if judgement_data else None
1424
+ judge_failure_reason = judgement_data.get('failure_reason') if judgement_data else None
1425
+
1426
+ self.telemetry.capture(
1427
+ AgentTelemetryEvent(
1428
+ task=self.task,
1429
+ model=self.llm.model,
1430
+ model_provider=self.llm.provider,
1431
+ max_steps=max_steps,
1432
+ max_actions_per_step=self.settings.max_actions_per_step,
1433
+ use_vision=self.settings.use_vision,
1434
+ version=self.version,
1435
+ source=self.source,
1436
+ cdp_url=urlparse(self.browser_session.cdp_url).hostname
1437
+ if self.browser_session and self.browser_session.cdp_url
1438
+ else None,
1439
+ agent_type=None, # Regular Agent (not code-use)
1440
+ action_errors=self.history.errors(),
1441
+ action_history=action_history_data,
1442
+ urls_visited=self.history.urls(),
1443
+ steps=self.state.n_steps,
1444
+ total_input_tokens=token_summary.prompt_tokens,
1445
+ total_output_tokens=token_summary.completion_tokens,
1446
+ prompt_cached_tokens=token_summary.prompt_cached_tokens,
1447
+ total_tokens=token_summary.total_tokens,
1448
+ total_duration_seconds=self.history.total_duration_seconds(),
1449
+ success=self.history.is_successful(),
1450
+ final_result_response=final_result_str,
1451
+ error_message=agent_run_error,
1452
+ judge_verdict=judge_verdict,
1453
+ judge_reasoning=judge_reasoning,
1454
+ judge_failure_reason=judge_failure_reason,
1455
+ )
1456
+ )
1457
+
1458
+ async def take_step(self, step_info: AgentStepInfo | None = None) -> tuple[bool, bool]:
1459
+ """Take a step
1460
+
1461
+ Returns:
1462
+ Tuple[bool, bool]: (is_done, is_valid)
1463
+ """
1464
+ if step_info is not None and step_info.step_number == 0:
1465
+ # First step
1466
+ self._log_first_step_startup()
1467
+ # Normally there was no try catch here but the callback can raise an InterruptedError which we skip
1468
+ try:
1469
+ await self._execute_initial_actions()
1470
+ except InterruptedError:
1471
+ pass
1472
+ except Exception as e:
1473
+ raise e
1474
+
1475
+ await self.step(step_info)
1476
+
1477
+ if self.history.is_done():
1478
+ await self.log_completion()
1479
+ if self.register_done_callback:
1480
+ if inspect.iscoroutinefunction(self.register_done_callback):
1481
+ await self.register_done_callback(self.history)
1482
+ else:
1483
+ self.register_done_callback(self.history)
1484
+ return True, True
1485
+
1486
+ return False, False
1487
+
1488
+ def _extract_start_url(self, task: str) -> str | None:
1489
+ """Extract URL from task string using naive pattern matching."""
1490
+
1491
+ import re
1492
+
1493
+ # Remove email addresses from task before looking for URLs
1494
+ task_without_emails = re.sub(r'\b[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Z|a-z]{2,}\b', '', task)
1495
+
1496
+ # Look for common URL patterns
1497
+ patterns = [
1498
+ r'https?://[^\s<>"\']+', # Full URLs with http/https
1499
+ r'(?:www\.)?[a-zA-Z0-9-]+(?:\.[a-zA-Z0-9-]+)*\.[a-zA-Z]{2,}(?:/[^\s<>"\']*)?', # Domain names with subdomains and optional paths
1500
+ ]
1501
+
1502
+ # File extensions that should be excluded from URL detection
1503
+ # These are likely files rather than web pages to navigate to
1504
+ excluded_extensions = {
1505
+ # Documents
1506
+ 'pdf',
1507
+ 'doc',
1508
+ 'docx',
1509
+ 'xls',
1510
+ 'xlsx',
1511
+ 'ppt',
1512
+ 'pptx',
1513
+ 'odt',
1514
+ 'ods',
1515
+ 'odp',
1516
+ # Text files
1517
+ 'txt',
1518
+ 'md',
1519
+ 'csv',
1520
+ 'json',
1521
+ 'xml',
1522
+ 'yaml',
1523
+ 'yml',
1524
+ # Archives
1525
+ 'zip',
1526
+ 'rar',
1527
+ '7z',
1528
+ 'tar',
1529
+ 'gz',
1530
+ 'bz2',
1531
+ 'xz',
1532
+ # Images
1533
+ 'jpg',
1534
+ 'jpeg',
1535
+ 'png',
1536
+ 'gif',
1537
+ 'bmp',
1538
+ 'svg',
1539
+ 'webp',
1540
+ 'ico',
1541
+ # Audio/Video
1542
+ 'mp3',
1543
+ 'mp4',
1544
+ 'avi',
1545
+ 'mkv',
1546
+ 'mov',
1547
+ 'wav',
1548
+ 'flac',
1549
+ 'ogg',
1550
+ # Code/Data
1551
+ 'py',
1552
+ 'js',
1553
+ 'css',
1554
+ 'java',
1555
+ 'cpp',
1556
+ # Academic/Research
1557
+ 'bib',
1558
+ 'bibtex',
1559
+ 'tex',
1560
+ 'latex',
1561
+ 'cls',
1562
+ 'sty',
1563
+ # Other common file types
1564
+ 'exe',
1565
+ 'msi',
1566
+ 'dmg',
1567
+ 'pkg',
1568
+ 'deb',
1569
+ 'rpm',
1570
+ 'iso',
1571
+ }
1572
+
1573
+ excluded_words = {
1574
+ 'never',
1575
+ 'dont',
1576
+ 'not',
1577
+ "don't",
1578
+ }
1579
+
1580
+ found_urls = []
1581
+ for pattern in patterns:
1582
+ matches = re.finditer(pattern, task_without_emails)
1583
+ for match in matches:
1584
+ url = match.group(0)
1585
+ original_position = match.start() # Store original position before URL modification
1586
+
1587
+ # Remove trailing punctuation that's not part of URLs
1588
+ url = re.sub(r'[.,;:!?()\[\]]+$', '', url)
1589
+
1590
+ # Check if URL ends with a file extension that should be excluded
1591
+ url_lower = url.lower()
1592
+ should_exclude = False
1593
+ for ext in excluded_extensions:
1594
+ if f'.{ext}' in url_lower:
1595
+ should_exclude = True
1596
+ break
1597
+
1598
+ if should_exclude:
1599
+ self.logger.debug(f'Excluding URL with file extension from auto-navigation: {url}')
1600
+ continue
1601
+
1602
+ # If in the 20 characters before the url position is a word in excluded_words skip to avoid "Never go to this url"
1603
+ context_start = max(0, original_position - 20)
1604
+ context_text = task_without_emails[context_start:original_position]
1605
+ if any(word.lower() in context_text.lower() for word in excluded_words):
1606
+ self.logger.debug(
1607
+ f'Excluding URL with word in excluded words from auto-navigation: {url} (context: "{context_text.strip()}")'
1608
+ )
1609
+ continue
1610
+
1611
+ # Add https:// if missing (after excluded words check to avoid position calculation issues)
1612
+ if not url.startswith(('http://', 'https://')):
1613
+ url = 'https://' + url
1614
+
1615
+ found_urls.append(url)
1616
+
1617
+ unique_urls = list(set(found_urls))
1618
+ # If multiple URLs found, skip directly_open_urling
1619
+ if len(unique_urls) > 1:
1620
+ self.logger.debug(f'Multiple URLs found ({len(found_urls)}), skipping directly_open_url to avoid ambiguity')
1621
+ return None
1622
+
1623
+ # If exactly one URL found, return it
1624
+ if len(unique_urls) == 1:
1625
+ return unique_urls[0]
1626
+
1627
+ return None
1628
+
1629
+ async def _execute_step(
1630
+ self,
1631
+ step: int,
1632
+ max_steps: int,
1633
+ step_info: AgentStepInfo,
1634
+ on_step_start: AgentHookFunc | None = None,
1635
+ on_step_end: AgentHookFunc | None = None,
1636
+ ) -> bool:
1637
+ """
1638
+ Execute a single step with timeout.
1639
+
1640
+ Returns:
1641
+ bool: True if task is done, False otherwise
1642
+ """
1643
+ if on_step_start is not None:
1644
+ await on_step_start(self)
1645
+
1646
+ self.logger.debug(f'🚶 Starting step {step + 1}/{max_steps}...')
1647
+
1648
+ try:
1649
+ await asyncio.wait_for(
1650
+ self.step(step_info),
1651
+ timeout=180, # 3 minute timeout
1652
+ )
1653
+ self.logger.debug(f'✅ Completed step {step + 1}/{max_steps}')
1654
+ except TimeoutError:
1655
+ # Handle step timeout gracefully
1656
+ error_msg = f'Step {step + 1} timed out after 180 seconds'
1657
+ self.logger.error(f'⏰ {error_msg}')
1658
+ self.state.consecutive_failures += 1
1659
+ self.state.last_result = [ActionResult(error=error_msg)]
1660
+
1661
+ if on_step_end is not None:
1662
+ await on_step_end(self)
1663
+
1664
+ if self.history.is_done():
1665
+ await self.log_completion()
1666
+
1667
+ if self.register_done_callback:
1668
+ if inspect.iscoroutinefunction(self.register_done_callback):
1669
+ await self.register_done_callback(self.history)
1670
+ else:
1671
+ self.register_done_callback(self.history)
1672
+
1673
+ return True
1674
+
1675
+ return False
1676
+
1677
+ @observe(name='agent.run', ignore_input=True, ignore_output=True)
1678
+ @time_execution_async('--run')
1679
+ async def run(
1680
+ self,
1681
+ max_steps: int = 100,
1682
+ on_step_start: AgentHookFunc | None = None,
1683
+ on_step_end: AgentHookFunc | None = None,
1684
+ ) -> AgentHistoryList[AgentStructuredOutput]:
1685
+ """Execute the task with maximum number of steps"""
1686
+
1687
+ loop = asyncio.get_event_loop()
1688
+ agent_run_error: str | None = None # Initialize error tracking variable
1689
+ self._force_exit_telemetry_logged = False # ADDED: Flag for custom telemetry on force exit
1690
+
1691
+ # Set up the signal handler with callbacks specific to this agent
1692
+ from browser_use.utils import SignalHandler
1693
+
1694
+ # Define the custom exit callback function for second CTRL+C
1695
+ def on_force_exit_log_telemetry():
1696
+ self._log_agent_event(max_steps=max_steps, agent_run_error='SIGINT: Cancelled by user')
1697
+ # NEW: Call the flush method on the telemetry instance
1698
+ if hasattr(self, 'telemetry') and self.telemetry:
1699
+ self.telemetry.flush()
1700
+ self._force_exit_telemetry_logged = True # Set the flag
1701
+
1702
+ signal_handler = SignalHandler(
1703
+ loop=loop,
1704
+ pause_callback=self.pause,
1705
+ resume_callback=self.resume,
1706
+ custom_exit_callback=on_force_exit_log_telemetry, # Pass the new telemetrycallback
1707
+ exit_on_second_int=True,
1708
+ )
1709
+ signal_handler.register()
1710
+
1711
+ try:
1712
+ await self._log_agent_run()
1713
+
1714
+ self.logger.debug(
1715
+ f'🔧 Agent setup: Agent Session ID {self.session_id[-4:]}, Task ID {self.task_id[-4:]}, Browser Session ID {self.browser_session.id[-4:] if self.browser_session else "None"} {"(connecting via CDP)" if (self.browser_session and self.browser_session.cdp_url) else "(launching local browser)"}'
1716
+ )
1717
+
1718
+ # Initialize timing for session and task
1719
+ self._session_start_time = time.time()
1720
+ self._task_start_time = self._session_start_time # Initialize task start time
1721
+
1722
+ # Only dispatch session events if this is the first run
1723
+ if not self.state.session_initialized:
1724
+ self.logger.debug('📡 Dispatching CreateAgentSessionEvent...')
1725
+ # Emit CreateAgentSessionEvent at the START of run()
1726
+ self.eventbus.dispatch(CreateAgentSessionEvent.from_agent(self))
1727
+
1728
+ self.state.session_initialized = True
1729
+
1730
+ self.logger.debug('📡 Dispatching CreateAgentTaskEvent...')
1731
+ # Emit CreateAgentTaskEvent at the START of run()
1732
+ self.eventbus.dispatch(CreateAgentTaskEvent.from_agent(self))
1733
+
1734
+ # Log startup message on first step (only if we haven't already done steps)
1735
+ self._log_first_step_startup()
1736
+ # Start browser session and attach watchdogs
1737
+ await self.browser_session.start()
1738
+
1739
+ # Normally there was no try catch here but the callback can raise an InterruptedError
1740
+ try:
1741
+ await self._execute_initial_actions()
1742
+ except InterruptedError:
1743
+ pass
1744
+ except Exception as e:
1745
+ raise e
1746
+
1747
+ self.logger.debug(f'🔄 Starting main execution loop with max {max_steps} steps...')
1748
+ for step in range(max_steps):
1749
+ # Use the consolidated pause state management
1750
+ if self.state.paused:
1751
+ self.logger.debug(f'⏸️ Step {step}: Agent paused, waiting to resume...')
1752
+ await self._external_pause_event.wait()
1753
+ signal_handler.reset()
1754
+
1755
+ # Check if we should stop due to too many failures, if final_response_after_failure is True, we try one last time
1756
+ if (self.state.consecutive_failures) >= self.settings.max_failures + int(
1757
+ self.settings.final_response_after_failure
1758
+ ):
1759
+ self.logger.error(f'❌ Stopping due to {self.settings.max_failures} consecutive failures')
1760
+ agent_run_error = f'Stopped due to {self.settings.max_failures} consecutive failures'
1761
+ break
1762
+
1763
+ # Check control flags before each step
1764
+ if self.state.stopped:
1765
+ self.logger.info('🛑 Agent stopped')
1766
+ agent_run_error = 'Agent stopped programmatically'
1767
+ break
1768
+
1769
+ step_info = AgentStepInfo(step_number=step, max_steps=max_steps)
1770
+ is_done = await self._execute_step(step, max_steps, step_info, on_step_start, on_step_end)
1771
+
1772
+ if is_done:
1773
+ # Agent has marked the task as done
1774
+ if self.settings.use_judge:
1775
+ await self._judge_and_log()
1776
+ break
1777
+ else:
1778
+ agent_run_error = 'Failed to complete task in maximum steps'
1779
+
1780
+ self.history.add_item(
1781
+ AgentHistory(
1782
+ model_output=None,
1783
+ result=[ActionResult(error=agent_run_error, include_in_memory=True)],
1784
+ state=BrowserStateHistory(
1785
+ url='',
1786
+ title='',
1787
+ tabs=[],
1788
+ interacted_element=[],
1789
+ screenshot_path=None,
1790
+ ),
1791
+ metadata=None,
1792
+ )
1793
+ )
1794
+
1795
+ self.logger.info(f'❌ {agent_run_error}')
1796
+
1797
+ self.history.usage = await self.token_cost_service.get_usage_summary()
1798
+
1799
+ # set the model output schema and call it on the fly
1800
+ if self.history._output_model_schema is None and self.output_model_schema is not None:
1801
+ self.history._output_model_schema = self.output_model_schema
1802
+
1803
+ return self.history
1804
+
1805
+ except KeyboardInterrupt:
1806
+ # Already handled by our signal handler, but catch any direct KeyboardInterrupt as well
1807
+ self.logger.debug('Got KeyboardInterrupt during execution, returning current history')
1808
+ agent_run_error = 'KeyboardInterrupt'
1809
+
1810
+ self.history.usage = await self.token_cost_service.get_usage_summary()
1811
+
1812
+ return self.history
1813
+
1814
+ except Exception as e:
1815
+ self.logger.error(f'Agent run failed with exception: {e}', exc_info=True)
1816
+ agent_run_error = str(e)
1817
+ raise e
1818
+
1819
+ finally:
1820
+ # Log token usage summary
1821
+ await self.token_cost_service.log_usage_summary()
1822
+
1823
+ # Unregister signal handlers before cleanup
1824
+ signal_handler.unregister()
1825
+
1826
+ if not self._force_exit_telemetry_logged: # MODIFIED: Check the flag
1827
+ try:
1828
+ self._log_agent_event(max_steps=max_steps, agent_run_error=agent_run_error)
1829
+ except Exception as log_e: # Catch potential errors during logging itself
1830
+ self.logger.error(f'Failed to log telemetry event: {log_e}', exc_info=True)
1831
+ else:
1832
+ # ADDED: Info message when custom telemetry for SIGINT was already logged
1833
+ self.logger.debug('Telemetry for force exit (SIGINT) was logged by custom exit callback.')
1834
+
1835
+ # NOTE: CreateAgentSessionEvent and CreateAgentTaskEvent are now emitted at the START of run()
1836
+ # to match backend requirements for CREATE events to be fired when entities are created,
1837
+ # not when they are completed
1838
+
1839
+ # Emit UpdateAgentTaskEvent at the END of run() with final task state
1840
+ self.eventbus.dispatch(UpdateAgentTaskEvent.from_agent(self))
1841
+
1842
+ # Generate GIF if needed before stopping event bus
1843
+ if self.settings.generate_gif:
1844
+ output_path: str = 'agent_history.gif'
1845
+ if isinstance(self.settings.generate_gif, str):
1846
+ output_path = self.settings.generate_gif
1847
+
1848
+ # Lazy import gif module to avoid heavy startup cost
1849
+ from browser_use.agent.gif import create_history_gif
1850
+
1851
+ create_history_gif(task=self.task, history=self.history, output_path=output_path)
1852
+
1853
+ # Only emit output file event if GIF was actually created
1854
+ if Path(output_path).exists():
1855
+ output_event = await CreateAgentOutputFileEvent.from_agent_and_file(self, output_path)
1856
+ self.eventbus.dispatch(output_event)
1857
+
1858
+ # Log final messages to user based on outcome
1859
+ self._log_final_outcome_messages()
1860
+
1861
+ # Stop the event bus gracefully, waiting for all events to be processed
1862
+ # Use longer timeout to avoid deadlocks in tests with multiple agents
1863
+ await self.eventbus.stop(timeout=3.0)
1864
+
1865
+ await self.close()
1866
+
1867
+ @observe_debug(ignore_input=True, ignore_output=True)
1868
+ @time_execution_async('--multi_act')
1869
+ async def multi_act(self, actions: list[ActionModel]) -> list[ActionResult]:
1870
+ """Execute multiple actions"""
1871
+ results: list[ActionResult] = []
1872
+ time_elapsed = 0
1873
+ total_actions = len(actions)
1874
+
1875
+ assert self.browser_session is not None, 'BrowserSession is not set up'
1876
+ try:
1877
+ if (
1878
+ self.browser_session._cached_browser_state_summary is not None
1879
+ and self.browser_session._cached_browser_state_summary.dom_state is not None
1880
+ ):
1881
+ cached_selector_map = dict(self.browser_session._cached_browser_state_summary.dom_state.selector_map)
1882
+ cached_element_hashes = {e.parent_branch_hash() for e in cached_selector_map.values()}
1883
+ else:
1884
+ cached_selector_map = {}
1885
+ cached_element_hashes = set()
1886
+ except Exception as e:
1887
+ self.logger.error(f'Error getting cached selector map: {e}')
1888
+ cached_selector_map = {}
1889
+ cached_element_hashes = set()
1890
+
1891
+ for i, action in enumerate(actions):
1892
+ if i > 0:
1893
+ # ONLY ALLOW TO CALL `done` IF IT IS A SINGLE ACTION
1894
+ if action.model_dump(exclude_unset=True).get('done') is not None:
1895
+ msg = f'Done action is allowed only as a single action - stopped after action {i} / {total_actions}.'
1896
+ self.logger.debug(msg)
1897
+ break
1898
+
1899
+ # wait between actions (only after first action)
1900
+ if i > 0:
1901
+ self.logger.debug(f'Waiting {self.browser_profile.wait_between_actions} seconds between actions')
1902
+ await asyncio.sleep(self.browser_profile.wait_between_actions)
1903
+
1904
+ try:
1905
+ await self._check_stop_or_pause()
1906
+ # Get action name from the action model
1907
+ action_data = action.model_dump(exclude_unset=True)
1908
+ action_name = next(iter(action_data.keys())) if action_data else 'unknown'
1909
+
1910
+ # Log action before execution
1911
+ self._log_action(action, action_name, i + 1, total_actions)
1912
+
1913
+ time_start = time.time()
1914
+
1915
+ result = await self.tools.act(
1916
+ action=action,
1917
+ browser_session=self.browser_session,
1918
+ file_system=self.file_system,
1919
+ page_extraction_llm=self.settings.page_extraction_llm,
1920
+ sensitive_data=self.sensitive_data,
1921
+ available_file_paths=self.available_file_paths,
1922
+ )
1923
+
1924
+ time_end = time.time()
1925
+ time_elapsed = time_end - time_start
1926
+
1927
+ results.append(result)
1928
+
1929
+ if results[-1].is_done or results[-1].error or i == total_actions - 1:
1930
+ break
1931
+
1932
+ except Exception as e:
1933
+ # Handle any exceptions during action execution
1934
+ self.logger.error(f'❌ Executing action {i + 1} failed -> {type(e).__name__}: {e}')
1935
+ raise e
1936
+
1937
+ return results
1938
+
1939
+ def _log_action(self, action, action_name: str, action_num: int, total_actions: int) -> None:
1940
+ """Log the action before execution with colored formatting"""
1941
+ # Color definitions
1942
+ blue = '\033[34m' # Action name
1943
+ magenta = '\033[35m' # Parameter names
1944
+ reset = '\033[0m'
1945
+
1946
+ # Format action number and name
1947
+ if total_actions > 1:
1948
+ action_header = f'▶️ [{action_num}/{total_actions}] {blue}{action_name}{reset}:'
1949
+ else:
1950
+ action_header = f'▶️ {blue}{action_name}{reset}:'
1951
+
1952
+ # Get action parameters
1953
+ action_data = action.model_dump(exclude_unset=True)
1954
+ params = action_data.get(action_name, {})
1955
+
1956
+ # Build parameter parts with colored formatting
1957
+ param_parts = []
1958
+
1959
+ if params and isinstance(params, dict):
1960
+ for param_name, value in params.items():
1961
+ # Truncate long values for readability
1962
+ if isinstance(value, str) and len(value) > 150:
1963
+ display_value = value[:150] + '...'
1964
+ elif isinstance(value, list) and len(str(value)) > 200:
1965
+ display_value = str(value)[:200] + '...'
1966
+ else:
1967
+ display_value = value
1968
+
1969
+ param_parts.append(f'{magenta}{param_name}{reset}: {display_value}')
1970
+
1971
+ # Join all parts
1972
+ if param_parts:
1973
+ params_string = ', '.join(param_parts)
1974
+ self.logger.info(f' {action_header} {params_string}')
1975
+ else:
1976
+ self.logger.info(f' {action_header}')
1977
+
1978
+ async def log_completion(self) -> None:
1979
+ """Log the completion of the task"""
1980
+ # self._task_end_time = time.time()
1981
+ # self._task_duration = self._task_end_time - self._task_start_time TODO: this is not working when using take_step
1982
+ if self.history.is_successful():
1983
+ self.logger.info('✅ Task completed successfully')
1984
+
1985
+ async def rerun_history(
1986
+ self,
1987
+ history: AgentHistoryList,
1988
+ max_retries: int = 3,
1989
+ skip_failures: bool = True,
1990
+ delay_between_actions: float = 2.0,
1991
+ ) -> list[ActionResult]:
1992
+ """
1993
+ Rerun a saved history of actions with error handling and retry logic.
1994
+
1995
+ Args:
1996
+ history: The history to replay
1997
+ max_retries: Maximum number of retries per action
1998
+ skip_failures: Whether to skip failed actions or stop execution
1999
+ delay_between_actions: Delay between actions in seconds
2000
+
2001
+ Returns:
2002
+ List of action results
2003
+ """
2004
+ # Skip cloud sync session events for rerunning (we're replaying, not starting new)
2005
+ self.state.session_initialized = True
2006
+
2007
+ # Initialize browser session
2008
+ await self.browser_session.start()
2009
+
2010
+ results = []
2011
+
2012
+ for i, history_item in enumerate(history.history):
2013
+ goal = history_item.model_output.current_state.next_goal if history_item.model_output else ''
2014
+ step_num = history_item.metadata.step_number if history_item.metadata else i
2015
+ step_name = 'Initial actions' if step_num == 0 else f'Step {step_num}'
2016
+ self.logger.info(f'Replaying {step_name} ({i + 1}/{len(history.history)}): {goal}')
2017
+
2018
+ if (
2019
+ not history_item.model_output
2020
+ or not history_item.model_output.action
2021
+ or history_item.model_output.action == [None]
2022
+ ):
2023
+ self.logger.warning(f'{step_name}: No action to replay, skipping')
2024
+ results.append(ActionResult(error='No action to replay'))
2025
+ continue
2026
+
2027
+ retry_count = 0
2028
+ while retry_count < max_retries:
2029
+ try:
2030
+ result = await self._execute_history_step(history_item, delay_between_actions)
2031
+ results.extend(result)
2032
+ break
2033
+
2034
+ except Exception as e:
2035
+ retry_count += 1
2036
+ if retry_count == max_retries:
2037
+ error_msg = f'{step_name} failed after {max_retries} attempts: {str(e)}'
2038
+ self.logger.error(error_msg)
2039
+ if not skip_failures:
2040
+ results.append(ActionResult(error=error_msg))
2041
+ raise RuntimeError(error_msg)
2042
+ else:
2043
+ self.logger.warning(f'{step_name} failed (attempt {retry_count}/{max_retries}), retrying...')
2044
+ await asyncio.sleep(delay_between_actions)
2045
+
2046
+ await self.close()
2047
+ return results
2048
+
2049
+ async def _execute_initial_actions(self) -> None:
2050
+ # Execute initial actions if provided
2051
+ if self.initial_actions and not self.state.follow_up_task:
2052
+ self.logger.debug(f'⚡ Executing {len(self.initial_actions)} initial actions...')
2053
+ result = await self.multi_act(self.initial_actions)
2054
+ # update result 1 to mention that its was automatically loaded
2055
+ if result and self.initial_url and result[0].long_term_memory:
2056
+ result[0].long_term_memory = f'Found initial url and automatically loaded it. {result[0].long_term_memory}'
2057
+ self.state.last_result = result
2058
+
2059
+ # Save initial actions to history as step 0 for rerun capability
2060
+ # Skip browser state capture for initial actions (usually just URL navigation)
2061
+ if self.settings.flash_mode:
2062
+ model_output = self.AgentOutput(
2063
+ evaluation_previous_goal=None,
2064
+ memory='Initial navigation',
2065
+ next_goal=None,
2066
+ action=self.initial_actions,
2067
+ )
2068
+ else:
2069
+ model_output = self.AgentOutput(
2070
+ evaluation_previous_goal='Start',
2071
+ memory=None,
2072
+ next_goal='Initial navigation',
2073
+ action=self.initial_actions,
2074
+ )
2075
+
2076
+ metadata = StepMetadata(
2077
+ step_number=0,
2078
+ step_start_time=time.time(),
2079
+ step_end_time=time.time(),
2080
+ )
2081
+
2082
+ # Create minimal browser state history for initial actions
2083
+ state_history = BrowserStateHistory(
2084
+ url=self.initial_url or '',
2085
+ title='Initial Actions',
2086
+ tabs=[],
2087
+ interacted_element=[None] * len(self.initial_actions), # No DOM elements needed
2088
+ screenshot_path=None,
2089
+ )
2090
+
2091
+ history_item = AgentHistory(
2092
+ model_output=model_output,
2093
+ result=result,
2094
+ state=state_history,
2095
+ metadata=metadata,
2096
+ )
2097
+
2098
+ self.history.add_item(history_item)
2099
+ self.logger.debug('📝 Saved initial actions to history as step 0')
2100
+ self.logger.debug('Initial actions completed')
2101
+
2102
+ async def _execute_history_step(self, history_item: AgentHistory, delay: float) -> list[ActionResult]:
2103
+ """Execute a single step from history with element validation"""
2104
+ assert self.browser_session is not None, 'BrowserSession is not set up'
2105
+ state = await self.browser_session.get_browser_state_summary(include_screenshot=False)
2106
+ if not state or not history_item.model_output:
2107
+ raise ValueError('Invalid state or model output')
2108
+ updated_actions = []
2109
+ for i, action in enumerate(history_item.model_output.action):
2110
+ updated_action = await self._update_action_indices(
2111
+ history_item.state.interacted_element[i],
2112
+ action,
2113
+ state,
2114
+ )
2115
+ updated_actions.append(updated_action)
2116
+
2117
+ if updated_action is None:
2118
+ raise ValueError(f'Could not find matching element {i} in current page')
2119
+
2120
+ result = await self.multi_act(updated_actions)
2121
+
2122
+ await asyncio.sleep(delay)
2123
+ return result
2124
+
2125
+ async def _update_action_indices(
2126
+ self,
2127
+ historical_element: DOMInteractedElement | None,
2128
+ action: ActionModel, # Type this properly based on your action model
2129
+ browser_state_summary: BrowserStateSummary,
2130
+ ) -> ActionModel | None:
2131
+ """
2132
+ Update action indices based on current page state.
2133
+ Returns updated action or None if element cannot be found.
2134
+ """
2135
+ if not historical_element or not browser_state_summary.dom_state.selector_map:
2136
+ return action
2137
+
2138
+ # selector_hash_map = {hash(e): e for e in browser_state_summary.dom_state.selector_map.values()}
2139
+
2140
+ highlight_index, current_element = next(
2141
+ (
2142
+ (highlight_index, element)
2143
+ for highlight_index, element in browser_state_summary.dom_state.selector_map.items()
2144
+ if element.element_hash == historical_element.element_hash
2145
+ ),
2146
+ (None, None),
2147
+ )
2148
+
2149
+ if not current_element or highlight_index is None:
2150
+ return None
2151
+
2152
+ old_index = action.get_index()
2153
+ if old_index != highlight_index:
2154
+ action.set_index(highlight_index)
2155
+ self.logger.info(f'Element moved in DOM, updated index from {old_index} to {highlight_index}')
2156
+
2157
+ return action
2158
+
2159
+ async def load_and_rerun(self, history_file: str | Path | None = None, **kwargs) -> list[ActionResult]:
2160
+ """
2161
+ Load history from file and rerun it.
2162
+
2163
+ Args:
2164
+ history_file: Path to the history file
2165
+ **kwargs: Additional arguments passed to rerun_history
2166
+ """
2167
+ if not history_file:
2168
+ history_file = 'AgentHistory.json'
2169
+ history = AgentHistoryList.load_from_file(history_file, self.AgentOutput)
2170
+ return await self.rerun_history(history, **kwargs)
2171
+
2172
+ def save_history(self, file_path: str | Path | None = None) -> None:
2173
+ """Save the history to a file with sensitive data filtering"""
2174
+ if not file_path:
2175
+ file_path = 'AgentHistory.json'
2176
+ self.history.save_to_file(file_path, sensitive_data=self.sensitive_data)
2177
+
2178
+ def pause(self) -> None:
2179
+ """Pause the agent before the next step"""
2180
+ print('\n\n⏸️ Paused the agent and left the browser open.\n\tPress [Enter] to resume or [Ctrl+C] again to quit.')
2181
+ self.state.paused = True
2182
+ self._external_pause_event.clear()
2183
+
2184
+ def resume(self) -> None:
2185
+ """Resume the agent"""
2186
+ # TODO: Locally the browser got closed
2187
+ print('----------------------------------------------------------------------')
2188
+ print('▶️ Resuming agent execution where it left off...\n')
2189
+ self.state.paused = False
2190
+ self._external_pause_event.set()
2191
+
2192
+ def stop(self) -> None:
2193
+ """Stop the agent"""
2194
+ self.logger.info('⏹️ Agent stopping')
2195
+ self.state.stopped = True
2196
+
2197
+ # Signal pause event to unblock any waiting code so it can check the stopped state
2198
+ self._external_pause_event.set()
2199
+
2200
+ # Task stopped
2201
+
2202
+ def _convert_initial_actions(self, actions: list[dict[str, dict[str, Any]]]) -> list[ActionModel]:
2203
+ """Convert dictionary-based actions to ActionModel instances"""
2204
+ converted_actions = []
2205
+ action_model = self.ActionModel
2206
+ for action_dict in actions:
2207
+ # Each action_dict should have a single key-value pair
2208
+ action_name = next(iter(action_dict))
2209
+ params = action_dict[action_name]
2210
+
2211
+ # Get the parameter model for this action from registry
2212
+ action_info = self.tools.registry.registry.actions[action_name]
2213
+ param_model = action_info.param_model
2214
+
2215
+ # Create validated parameters using the appropriate param model
2216
+ validated_params = param_model(**params)
2217
+
2218
+ # Create ActionModel instance with the validated parameters
2219
+ action_model = self.ActionModel(**{action_name: validated_params})
2220
+ converted_actions.append(action_model)
2221
+
2222
+ return converted_actions
2223
+
2224
+ def _verify_and_setup_llm(self):
2225
+ """
2226
+ Verify that the LLM API keys are setup and the LLM API is responding properly.
2227
+ Also handles tool calling method detection if in auto mode.
2228
+ """
2229
+
2230
+ # Skip verification if already done
2231
+ if getattr(self.llm, '_verified_api_keys', None) is True or CONFIG.SKIP_LLM_API_KEY_VERIFICATION:
2232
+ setattr(self.llm, '_verified_api_keys', True)
2233
+ return True
2234
+
2235
+ @property
2236
+ def message_manager(self) -> MessageManager:
2237
+ return self._message_manager
2238
+
2239
+ async def close(self):
2240
+ """Close all resources"""
2241
+ try:
2242
+ # Only close browser if keep_alive is False (or not set)
2243
+ if self.browser_session is not None:
2244
+ if not self.browser_session.browser_profile.keep_alive:
2245
+ # Kill the browser session - this dispatches BrowserStopEvent,
2246
+ # stops the EventBus with clear=True, and recreates a fresh EventBus
2247
+ await self.browser_session.kill()
2248
+
2249
+ # Force garbage collection
2250
+ gc.collect()
2251
+
2252
+ # Debug: Log remaining threads and asyncio tasks
2253
+ import threading
2254
+
2255
+ threads = threading.enumerate()
2256
+ self.logger.debug(f'🧵 Remaining threads ({len(threads)}): {[t.name for t in threads]}')
2257
+
2258
+ # Get all asyncio tasks
2259
+ tasks = asyncio.all_tasks(asyncio.get_event_loop())
2260
+ # Filter out the current task (this close() coroutine)
2261
+ other_tasks = [t for t in tasks if t != asyncio.current_task()]
2262
+ if other_tasks:
2263
+ self.logger.debug(f'⚡ Remaining asyncio tasks ({len(other_tasks)}):')
2264
+ for task in other_tasks[:10]: # Limit to first 10 to avoid spam
2265
+ self.logger.debug(f' - {task.get_name()}: {task}')
2266
+
2267
+ except Exception as e:
2268
+ self.logger.error(f'Error during cleanup: {e}')
2269
+
2270
+ async def _update_action_models_for_page(self, page_url: str) -> None:
2271
+ """Update action models with page-specific actions"""
2272
+ # Create new action model with current page's filtered actions
2273
+ self.ActionModel = self.tools.registry.create_action_model(page_url=page_url)
2274
+ # Update output model with the new actions
2275
+ if self.settings.flash_mode:
2276
+ self.AgentOutput = AgentOutput.type_with_custom_actions_flash_mode(self.ActionModel)
2277
+ elif self.settings.use_thinking:
2278
+ self.AgentOutput = AgentOutput.type_with_custom_actions(self.ActionModel)
2279
+ else:
2280
+ self.AgentOutput = AgentOutput.type_with_custom_actions_no_thinking(self.ActionModel)
2281
+
2282
+ # Update done action model too
2283
+ self.DoneActionModel = self.tools.registry.create_action_model(include_actions=['done'], page_url=page_url)
2284
+ if self.settings.flash_mode:
2285
+ self.DoneAgentOutput = AgentOutput.type_with_custom_actions_flash_mode(self.DoneActionModel)
2286
+ elif self.settings.use_thinking:
2287
+ self.DoneAgentOutput = AgentOutput.type_with_custom_actions(self.DoneActionModel)
2288
+ else:
2289
+ self.DoneAgentOutput = AgentOutput.type_with_custom_actions_no_thinking(self.DoneActionModel)
2290
+
2291
+ async def authenticate_cloud_sync(self, show_instructions: bool = True) -> bool:
2292
+ """
2293
+ Authenticate with cloud service for future runs.
2294
+
2295
+ This is useful when users want to authenticate after a task has completed
2296
+ so that future runs will sync to the cloud.
2297
+
2298
+ Args:
2299
+ show_instructions: Whether to show authentication instructions to user
2300
+
2301
+ Returns:
2302
+ bool: True if authentication was successful
2303
+ """
2304
+ self.logger.warning('Cloud sync has been removed and is no longer available')
2305
+ return False
2306
+
2307
+ def run_sync(
2308
+ self,
2309
+ max_steps: int = 100,
2310
+ on_step_start: AgentHookFunc | None = None,
2311
+ on_step_end: AgentHookFunc | None = None,
2312
+ ) -> AgentHistoryList[AgentStructuredOutput]:
2313
+ """Synchronous wrapper around the async run method for easier usage without asyncio."""
2314
+ import asyncio
2315
+
2316
+ return asyncio.run(self.run(max_steps=max_steps, on_step_start=on_step_start, on_step_end=on_step_end))