vibesurf 0.1.21__py3-none-any.whl → 0.2.6__py3-none-any.whl

vibe_surf/_version.py

@@ -28,7 +28,7 @@ version_tuple: VERSION_TUPLE
 commit_id: COMMIT_ID
 __commit_id__: COMMIT_ID
 
-__version__ = version = '0.1.21'
-__version_tuple__ = version_tuple = (0, 1, 21)
+__version__ = version = '0.2.6'
+__version_tuple__ = version_tuple = (0, 2, 6)
 
 __commit_id__ = commit_id = None

vibe_surf/agents/browser_use_agent.py

@@ -64,7 +64,6 @@ from browser_use.dom.views import DOMInteractedElement
 from browser_use.filesystem.file_system import FileSystem
 from browser_use.observability import observe, observe_debug
 from browser_use.sync import CloudSync
-from browser_use.telemetry.service import ProductTelemetry
 from browser_use.telemetry.views import AgentTelemetryEvent
 from browser_use.utils import (
     _log_pretty_path,
@@ -73,9 +72,10 @@ from browser_use.utils import (
     time_execution_async,
     time_execution_sync,
 )
-
+from browser_use.llm.messages import BaseMessage, ContentPartImageParam, ContentPartTextParam, UserMessage
 from browser_use.agent.service import Agent, AgentHookFunc
 from vibe_surf.tools.file_system import CustomFileSystem
+from vibe_surf.telemetry.service import ProductTelemetry
 
 Context = TypeVar('Context')
 
@@ -107,6 +107,7 @@ class BrowserUseAgent(Agent):
                     | None
             ) = None,
             register_external_agent_status_raise_error_callback: Callable[[], Awaitable[bool]] | None = None,
+            register_should_stop_callback: Callable[[], Awaitable[bool]] | None = None,
             # Agent settings
             output_model_schema: type[AgentStructuredOutput] | None = None,
             use_vision: bool = True,
@@ -127,7 +128,6 @@ class BrowserUseAgent(Agent):
             source: str | None = None,
             file_system_path: str | None = None,
             task_id: str | None = None,
-            cloud_sync: CloudSync | None = None,
             calculate_cost: bool = False,
             display_files_in_done_text: bool = True,
             include_tool_call_examples: bool = False,
@@ -136,7 +136,9 @@ class BrowserUseAgent(Agent):
             step_timeout: int = 120,
             directly_open_url: bool = False,
             include_recent_events: bool = False,
-            allow_parallel_action_types: list[str] = ["extract_structured_data", "extract_content_from_file"],
+            sample_images: list[ContentPartTextParam | ContentPartImageParam] | None = None,
+            final_response_after_failure: bool = True,
+            allow_parallel_action_types: list[str] = ["extract", "extract_content_from_file"],
             _url_shortening_limit: int = 25,
             token_cost_service: Optional[TokenCost] = None,
             **kwargs,
@@ -151,7 +153,7 @@ class BrowserUseAgent(Agent):
         self.session_id: str = uuid7str()
         self.allow_parallel_action_types = allow_parallel_action_types
         self._url_shortening_limit = _url_shortening_limit
-
+        self.sample_images = sample_images
         browser_profile = browser_profile or DEFAULT_BROWSER_PROFILE
 
         # Handle browser vs browser_session parameter (browser takes precedence)
@@ -206,6 +208,7 @@ class BrowserUseAgent(Agent):
             include_tool_call_examples=include_tool_call_examples,
             llm_timeout=llm_timeout,
             step_timeout=step_timeout,
+            final_response_after_failure=final_response_after_failure,
         )
 
         # Token cost service
@@ -289,7 +292,6 @@ class BrowserUseAgent(Agent):
         self._message_manager = MessageManager(
             task=task,
             system_message=SystemPrompt(
-                action_description=self.unfiltered_actions,
                 max_actions_per_step=self.settings.max_actions_per_step,
                 override_system_message=override_system_message,
                 extend_system_message=extend_system_message,
@@ -306,6 +308,7 @@ class BrowserUseAgent(Agent):
             vision_detail_level=self.settings.vision_detail_level,
             include_tool_call_examples=self.settings.include_tool_call_examples,
             include_recent_events=self.include_recent_events,
+            sample_images=self.sample_images,
         )
 
         if self.sensitive_data:
@@ -375,6 +378,7 @@ class BrowserUseAgent(Agent):
         # Callbacks
         self.register_new_step_callback = register_new_step_callback
         self.register_done_callback = register_done_callback
+        self.register_should_stop_callback = register_should_stop_callback
         self.register_external_agent_status_raise_error_callback = register_external_agent_status_raise_error_callback
 
         # Telemetry
@@ -463,7 +467,7 @@ class BrowserUseAgent(Agent):
 
             # Use _make_history_item like main branch
             await self._make_history_item(self.state.last_model_output, browser_state_summary, self.state.last_result,
-                                          metadata)
+                                          metadata, state_message=self._message_manager.last_state_message_text,)
 
         # Log step completion summary
         self._log_step_completion_summary(self.step_start_time, self.state.last_result)
@@ -490,6 +494,9 @@ class BrowserUseAgent(Agent):
         self.task = new_task
         self._message_manager.add_new_task(new_task)
 
+        # Mark as follow-up task and recreate eventbus (gets shut down after each run)
+        self.state.follow_up_task = True
+
     @observe(name='agent.run', metadata={'task': '{{task}}', 'debug': '{{debug}}'})
     @time_execution_async('--run')
     async def run(
@@ -525,7 +532,7 @@ class BrowserUseAgent(Agent):
         signal_handler.register()
 
         try:
-            self._log_agent_run()
+            await self._log_agent_run()
 
             self.logger.debug(
                 f'🔧 Agent setup: Task ID {self.task_id[-4:]}, Session ID {self.session_id[-4:]}, Browser Session ID {self.browser_session.id[-4:] if self.browser_session else "None"}'
@@ -673,349 +680,4 @@ class BrowserUseAgent(Agent):
 
                 create_history_gif(task=self.task, history=self.history, output_path=output_path)
 
-            await self.close()
-
-    def _matches_action_type(self, action_type: str, allowed_pattern: str) -> bool:
-        """
-        Check if an action type matches an allowed pattern, supporting wildcards.
-        
-        Args:
-            action_type: The actual action type (e.g., "mcp.filesystem.read_file")
-            allowed_pattern: The pattern to match (e.g., "mcp.filesystem*")
-            
-        Returns:
-            True if the action type matches the pattern
-        """
-        if allowed_pattern.endswith('*'):
-            # Wildcard matching
-            prefix = allowed_pattern[:-1]
-            return action_type.startswith(prefix)
-        else:
-            # Exact matching
-            return action_type == allowed_pattern
-
-    def _is_action_parallel_allowed(self, action: ActionModel) -> bool:
-        """
-        Check if an action is allowed to be executed in parallel.
-        
-        Args:
-            action: The action to check
-            
-        Returns:
-            True if the action can be executed in parallel
-        """
-        action_data = action.model_dump(exclude_unset=True)
-        action_type = next(iter(action_data.keys())) if action_data else None
-
-        if not action_type:
-            return False
-
-        for allowed_pattern in self.allow_parallel_action_types:
-            if self._matches_action_type(action_type, allowed_pattern):
-                return True
-
-        return False
-
-    def _group_actions_for_parallel_execution(self, actions: list[ActionModel]) -> list[list[ActionModel]]:
-        """
-        Group consecutive actions that can be executed in parallel.
-        
-        Args:
-            actions: List of actions to group
-            
-        Returns:
-            List of action groups, where each group can be executed in parallel
-        """
-        if not actions:
-            return []
-
-        groups = []
-        current_group = [actions[0]]
-
-        for i in range(1, len(actions)):
-            current_action = actions[i]
-            previous_action = actions[i - 1]
-
-            # Check if both current and previous actions can be executed in parallel
-            if (self._is_action_parallel_allowed(current_action) and
-                    self._is_action_parallel_allowed(previous_action)):
-                # Add to current group
-                current_group.append(current_action)
-            else:
-                # Start a new group
-                groups.append(current_group)
-                current_group = [current_action]
-
-        # Add the last group
-        groups.append(current_group)
-
-        return groups
-
-    @observe_debug(ignore_input=True, ignore_output=True)
-    @time_execution_async('--multi_act')
-    async def multi_act(
-            self,
-            actions: list[ActionModel],
-            check_for_new_elements: bool = True,
-    ) -> list[ActionResult]:
-        """Execute multiple actions, with parallel execution for allowed action types"""
-        results: list[ActionResult] = []
-        time_elapsed = 0
-        total_actions = len(actions)
-
-        assert self.browser_session is not None, 'BrowserSession is not set up'
-        try:
-            if (
-                    self.browser_session._cached_browser_state_summary is not None
-                    and self.browser_session._cached_browser_state_summary.dom_state is not None
-            ):
-                cached_selector_map = dict(self.browser_session._cached_browser_state_summary.dom_state.selector_map)
-                cached_element_hashes = {e.parent_branch_hash() for e in cached_selector_map.values()}
-            else:
-                cached_selector_map = {}
-                cached_element_hashes = set()
-        except Exception as e:
-            self.logger.error(f'Error getting cached selector map: {e}')
-            cached_selector_map = {}
-            cached_element_hashes = set()
-
-        # Group actions for potential parallel execution
-        action_groups = self._group_actions_for_parallel_execution(actions)
-
-        # Track global action index for logging and DOM checks
-        global_action_index = 0
-
-        for group_index, action_group in enumerate(action_groups):
-            group_size = len(action_group)
-
-            # Check if this group can be executed in parallel
-            can_execute_in_parallel = (
-                    group_size > 1 and
-                    all(self._is_action_parallel_allowed(action) for action in action_group)
-            )
-
-            if can_execute_in_parallel:
-                self.logger.info(
-                    f'🚀 Executing {group_size} actions in parallel: group {group_index + 1}/{len(action_groups)}')
-                # Execute actions in parallel using asyncio.gather
-                parallel_results = await self._execute_actions_in_parallel(
-                    action_group, global_action_index, total_actions,
-                    cached_selector_map, cached_element_hashes, check_for_new_elements
-                )
-                results.extend(parallel_results)
-                global_action_index += group_size
-
-                # Check if any result indicates completion or error
-                if any(result.is_done or result.error for result in parallel_results):
-                    break
-            else:
-                # Execute actions sequentially
-                for local_index, action in enumerate(action_group):
-                    i = global_action_index + local_index
-
-                    # Original sequential execution logic continues here...
-                    # if i > 0:
-                    #     # ONLY ALLOW TO CALL `done` IF IT IS A SINGLE ACTION
-                    #     if action.model_dump(exclude_unset=True).get('done') is not None:
-                    #         msg = f'Done action is allowed only as a single action - stopped after action {i} / {total_actions}.'
-                    #         self.logger.debug(msg)
-                    #         break
-
-                    # DOM synchronization check - verify element indexes are still valid AFTER first action
-                    if action.get_index() is not None and i != 0:
-                        result = await self._check_dom_synchronization(
-                            action, i, total_actions, cached_selector_map, cached_element_hashes,
-                            check_for_new_elements, actions
-                        )
-                        if result:
-                            results.append(result)
-                            break
-
-                    # wait between actions (only after first action)
-                    if i > 0:
-                        await asyncio.sleep(self.browser_profile.wait_between_actions)
-
-                    # Execute single action
-                    try:
-                        action_result = await self._execute_single_action(action, i, total_actions)
-                        results.append(action_result)
-
-                        if action_result.is_done or action_result.error or i == total_actions - 1:
-                            break
-
-                    except Exception as e:
-                        self.logger.error(f'❌ Executing action {i + 1} failed: {type(e).__name__}: {e}')
-                        raise e
-
-                global_action_index += len(action_group)
-
-        return results
-
-    async def _execute_actions_in_parallel(
-            self,
-            actions: list[ActionModel],
-            start_index: int,
-            total_actions: int,
-            cached_selector_map: dict,
-            cached_element_hashes: set,
-            check_for_new_elements: bool
-    ) -> list[ActionResult]:
-        """Execute a group of actions in parallel using asyncio.gather"""
-
-        async def execute_single_parallel_action(action: ActionModel, action_index: int) -> ActionResult:
-            """Execute a single action for parallel execution"""
-            await self._raise_if_stopped_or_paused()
-
-            # Get action info for logging
-            action_data = action.model_dump(exclude_unset=True)
-            action_name = next(iter(action_data.keys())) if action_data else 'unknown'
-            action_params = getattr(action, action_name, '') or str(action.model_dump(mode='json'))[:140].replace(
-                '"', ''
-            ).replace('{', '').replace('}', '').replace("'", '').strip().strip(',')
-            action_params = str(action_params)
-            action_params = f'{action_params[:122]}...' if len(action_params) > 128 else action_params
-
-            time_start = time.time()
-            blue = '\033[34m'
-            reset = '\033[0m'
-            self.logger.info(f'  🦾 {blue}[PARALLEL ACTION {action_index + 1}/{total_actions}]{reset} {action_params}')
-
-            # Execute the action
-            result = await self.tools.act(
-                action=action,
-                browser_session=self.browser_session,
-                file_system=self.file_system,
-                page_extraction_llm=self.settings.page_extraction_llm,
-                sensitive_data=self.sensitive_data,
-                available_file_paths=self.available_file_paths,
-            )
-
-            time_end = time.time()
-            time_elapsed = time_end - time_start
-
-            green = '\033[92m'
-            self.logger.debug(
-                f'☑️ Parallel action {action_index + 1}/{total_actions}: {green}{action_params}{reset} in {time_elapsed:.2f}s'
-            )
-
-            return result
-
-        # Create tasks for parallel execution
-        tasks = [
-            execute_single_parallel_action(action, start_index + i)
-            for i, action in enumerate(actions)
-        ]
-
-        # Execute all tasks in parallel
-        parallel_results = await asyncio.gather(*tasks, return_exceptions=True)
-
-        # Process results and handle any exceptions
-        processed_results = []
-        for i, result in enumerate(parallel_results):
-            if isinstance(result, Exception):
-                action_index = start_index + i
-                self.logger.error(f'❌ Parallel action {action_index + 1} failed: {type(result).__name__}: {result}')
-                raise result
-            else:
-                processed_results.append(result)
-
-        return processed_results
-
-    async def _check_dom_synchronization(
-            self,
-            action: ActionModel,
-            action_index: int,
-            total_actions: int,
-            cached_selector_map: dict,
-            cached_element_hashes: set,
-            check_for_new_elements: bool,
-            all_actions: list[ActionModel]
-    ) -> ActionResult | None:
-        """Check DOM synchronization and return result if page changed"""
-        new_browser_state_summary = await self.browser_session.get_browser_state_summary(
-            cache_clickable_elements_hashes=False,
-            include_screenshot=False,
-        )
-        new_selector_map = new_browser_state_summary.dom_state.selector_map
-
-        # Detect index change after previous action
-        orig_target = cached_selector_map.get(action.get_index())
-        orig_target_hash = orig_target.parent_branch_hash() if orig_target else None
-
-        new_target = new_selector_map.get(action.get_index())  # type: ignore
-        new_target_hash = new_target.parent_branch_hash() if new_target else None
-
-        def get_remaining_actions_str(actions: list[ActionModel], index: int) -> str:
-            remaining_actions = []
-            for remaining_action in actions[index:]:
-                action_data = remaining_action.model_dump(exclude_unset=True)
-                action_name = next(iter(action_data.keys())) if action_data else 'unknown'
-                remaining_actions.append(action_name)
-            return ', '.join(remaining_actions)
-
-        if orig_target_hash != new_target_hash:
-            # Get names of remaining actions that won't be executed
-            remaining_actions_str = get_remaining_actions_str(all_actions, action_index)
-            msg = f'Page changed after action {action_index} / {total_actions}: actions {remaining_actions_str} were not executed'
-            self.logger.info(msg)
-            return ActionResult(
-                extracted_content=msg,
-                include_in_memory=True,
-                long_term_memory=msg,
-            )
-
-        # Check for new elements that appeared
-        new_element_hashes = {e.parent_branch_hash() for e in new_selector_map.values()}
-        if check_for_new_elements and not new_element_hashes.issubset(cached_element_hashes):
-            # next action requires index but there are new elements on the page
-            remaining_actions_str = get_remaining_actions_str(all_actions, action_index)
-            msg = f'Something new appeared after action {action_index} / {total_actions}: actions {remaining_actions_str} were not executed'
-            self.logger.info(msg)
-            return ActionResult(
-                extracted_content=msg,
-                include_in_memory=True,
-                long_term_memory=msg,
-            )
-
-        return None
-
-    async def _execute_single_action(self, action: ActionModel, action_index: int, total_actions: int) -> ActionResult:
-        """Execute a single action in sequential mode"""
-        await self._raise_if_stopped_or_paused()
-
-        # Get action name from the action model
-        action_data = action.model_dump(exclude_unset=True)
-        action_name = next(iter(action_data.keys())) if action_data else 'unknown'
-        action_params = getattr(action, action_name, '') or str(action.model_dump(mode='json'))[:140].replace(
-            '"', ''
-        ).replace('{', '').replace('}', '').replace("'", '').strip().strip(',')
-        # Ensure action_params is always a string before checking length
-        action_params = str(action_params)
-        action_params = f'{action_params[:122]}...' if len(action_params) > 128 else action_params
-
-        time_start = time.time()
-
-        red = '\033[91m'
-        green = '\033[92m'
-        blue = '\033[34m'
-        reset = '\033[0m'
-
-        self.logger.info(f'  🦾 {blue}[ACTION {action_index + 1}/{total_actions}]{reset} {action_params}')
-
-        result = await self.tools.act(
-            action=action,
-            browser_session=self.browser_session,
-            file_system=self.file_system,
-            page_extraction_llm=self.settings.page_extraction_llm,
-            sensitive_data=self.sensitive_data,
-            available_file_paths=self.available_file_paths,
-        )
-
-        time_end = time.time()
-        time_elapsed = time_end - time_start
-
-        self.logger.debug(
-            f'☑️ Executed action {action_index + 1}/{total_actions}: {green}{action_params}{reset} in {time_elapsed:.2f}s'
-        )
-
-        return result
+            await self.close()

vibe_surf/agents/prompts/vibe_surf_prompt.py

@@ -65,6 +65,49 @@ You will receive contextual information including:
 - Include file references in task descriptions when relevant
 - All file operations automatically resolve relative to the workspace directory
 
+## Extra Tools Discovery and Usage
+
+When you cannot find suitable built-in tools to complete user requirements (excluding browser-related tasks), you can discover and use additional tools through the following workflow. Extra tools generally consist of Composio toolkit integrations and user-defined MCP servers.
+
+1. **Discover Available Toolkits**: Use `get_all_toolkit_types` to retrieve all available toolkit types from Composio and MCP integrations. You only need to call this once per session unless you need to refresh the toolkit list.
+
+2. **Search for Relevant Tools**: Use `search_tool` with the appropriate toolkit type and search filters to find tools that match the user's requirements. Provide specific keywords related to the functionality needed.
+
+3. **Get Tool Information**: Use `get_tool_info` to retrieve detailed parameter information for any tool you want to use. This will show you the exact parameter schema and requirements.
+
+4. **Execute the Tool**: Use `execute_extra_tool` with the tool name and properly formatted JSON parameters to execute the desired functionality.
+
+**Usage Guidelines:**
+- **Prioritize Composio Tools**: When available, prefer Composio toolkit tools over browser automation for API-based tasks (e.g., Gmail, GitHub, Google Calendar, Twitter,  Slack  and etc.) as they provide much higher efficiency through direct API calls
+- **Parameter Optimization**: Always optimize default parameters to prevent information overload. Use appropriate filters and limits to get only essential information. Such as: Set `include_payload=False` when possible to avoid unnecessary response data.
+
+This approach allows you to leverage a wide range of external integrations and APIs beyond the core browser automation capabilities.
+
+## Authentication Error Handling
+
+When using tools to fetch information from social media platforms (such as 小红书/XHS, 微博/Weibo, 抖音/Douyin, etc.), authentication errors may occur due to missing or expired credentials.
+
+**Authentication Error Response Protocol:**
+- **Direct User Notification**: When encountering authentication errors (401 Unauthorized, login required, token expired, etc.), immediately inform the user that they need to complete authentication or login
+- **Clear Guidance**: Provide clear instructions on what the user needs to do to resolve the authentication issue
+- **No Force Continuation**: Do not attempt to force other operations or workarounds when authentication is required
+- **Simple Language**: Use straightforward language like "Please complete authentication/login for [platform name] first" or "需要先完成[平台名称]的登录验证"
+
+**Example Response for Authentication Errors:**
+```
+Authentication required for [Platform Name]. Please complete login/authentication for the platform first, then try again.
+需要先完成[平台名称]的登录验证，请先登录后重试。
+```
+
+## Skills Command Processing
+- When users input commands in `/skill_name` format, please use the corresponding skill action:
+- **Tab Targeting[Optional]**: Such as `/crawl @1234` → Execute `skill_crawl` with tab_id "1234"
+- **Parameter Processing**: Sometimes user provide uncompleted or simple prompt, please convert it to correct and optimized params. Such as convert natural language to valid JavaScript for code skill
+- **Special Cases**: `skill_deep_research` only returns guidelines only, then follow guidelines to conduct actual research
+- **Execution Policy**: Skill actions execute only once (no need to retry if errors occur), and all results - whether successful or failed - should be presented to users in structured markdown format.
+- **Follow-up Operations**: When users input skill operations without specifying additional tasks, do not automatically perform subsequent operations. Only perform additional tool operations when users specifically request actions like saving results to files or writing reports.
+- **Search Skill Usage**: `/skill_search` should ONLY be used when users want to quickly obtain specific information or news and user specify `/skill_search` in request. Please analyze user intent carefully - if the request contains other browser tasks or requires more complex web operations, you should generally execute browser tasks instead of using skill search.
+
 ## Language Adaptability
 
 **Critical**: Your output language must match the user's request language. If the user communicates in Chinese, respond in Chinese. If in English, respond in English. Maintain consistency throughout the interaction.
@@ -77,6 +120,7 @@ Before executing any action:
 3. **Plan Resource Usage**: Consider tab management and session optimization
 4. **Validate Completeness**: Ensure all user requirements are addressed
 
+
 Execute with precision, leverage concurrent capabilities for efficiency, and deliver professional results that exceed expectations.
 """
 
@@ -86,4 +130,5 @@ EXTEND_BU_SYSTEM_PROMPT = """
 * Regarding file operations, please note that you need the full relative path (including subfolders), not just the file name.
 * Especially when a file operation reports an error, please reflect whether the file path is not written correctly, such as the subfolder is not written.
 * If you are operating on files in the filesystem, be sure to use relative paths (relative to the workspace dir) instead of absolute paths.
+* If you are typing in the search box, please use Enter key to search instead of clicking.
 """