camel-ai 0.2.73a12__py3-none-any.whl → 0.2.74__py3-none-any.whl

camel/toolkits/hybrid_browser_toolkit_py/unified_analyzer.js

@@ -1,4 +1,4 @@
-(() => {
+((viewport_limit = false) => {
     // Unified analyzer that combines visual and structural analysis
     // Preserves complete snapshot.js logic while adding visual coordinate information
 
@@ -406,6 +406,11 @@
         if (tagName === 'header') return 'banner';
         if (tagName === 'footer') return 'contentinfo';
         if (tagName === 'fieldset') return 'group';
+        
+        // Enhanced role mappings for table elements
+        if (tagName === 'table') return 'table';
+        if (tagName === 'tr') return 'row';
+        if (tagName === 'td' || tagName === 'th') return 'cell';
 
         return 'generic';
     }
@@ -484,6 +489,9 @@
 
         // Add a heuristic to ignore code-like text that might be in the DOM
         if ((text.match(/[;:{}]/g)?.length || 0) > 2) return '';
+        
+
+        
         return text;
         }
 
@@ -578,6 +586,8 @@
             const level = getAriaLevel(element);
             if (level > 0) node.level = level;
 
+
+            
             return node;
         }
 
@@ -725,6 +735,9 @@
         if (isRedundantWrapper) {
             return node.children;
         }
+        
+
+        
         return [node];
     }
 
@@ -815,6 +828,23 @@
 
     // === Visual analysis functions from page_script.js ===
 
+    // Check if element is within the current viewport
+    function isInViewport(element) {
+        if (!element || element.nodeType !== Node.ELEMENT_NODE) return false;
+        
+        try {
+            const rect = element.getBoundingClientRect();
+            return (
+                rect.top >= 0 &&
+                rect.left >= 0 &&
+                rect.bottom <= (window.innerHeight || document.documentElement.clientHeight) &&
+                rect.right <= (window.innerWidth || document.documentElement.clientWidth)
+            );
+        } catch (e) {
+            return false;
+        }
+    }
+
     // From page_script.js - check if element is topmost at coordinates
     function isTopmost(element, x, y) {
         let hit = document.elementFromPoint(x, y);
@@ -855,10 +885,21 @@
 
     // === Unified analysis function ===
 
-    function collectElementsFromTree(node, elementsMap) {
+    function collectElementsFromTree(node, elementsMap, viewportLimitEnabled = false) {
         if (typeof node === 'string') return;
 
         if (node.element && node.ref) {
+            // If viewport_limit is enabled, only include elements that are in the viewport
+            if (viewportLimitEnabled && !isInViewport(node.element)) {
+                // Skip this element but still process its children
+                if (node.children) {
+                    for (const child of node.children) {
+                        collectElementsFromTree(child, elementsMap, viewportLimitEnabled);
+                    }
+                }
+                return;
+            }
+
             // Get visual coordinates for this element
             const coordinates = getElementCoordinates(node.element);
 
@@ -891,7 +932,7 @@
         // Recursively process children
         if (node.children) {
             for (const child of node.children) {
-                collectElementsFromTree(child, elementsMap);
+                collectElementsFromTree(child, elementsMap, viewportLimitEnabled);
             }
         }
     }
@@ -931,7 +972,7 @@
         [tree] = normalizeTree(tree);
 
         const elementsMap = {};
-        collectElementsFromTree(tree, elementsMap);
+        collectElementsFromTree(tree, elementsMap, viewport_limit);
 
         // Verify uniqueness of aria-ref attributes (debugging aid)
         const ariaRefCounts = {};

camel/toolkits/note_taking_toolkit.py

@@ -138,7 +138,7 @@ class NoteTakingToolkit(BaseToolkit):
             self.registry.append(note_name)
             self._save_registry()
 
-    def create_note(self, note_name: str, content: str = "") -> str:
+    def create_note(self, note_name: str, content: str) -> str:
         r"""Creates a new note with a unique name.
 
         This function will create a new file for your note.
@@ -149,12 +149,11 @@ class NoteTakingToolkit(BaseToolkit):
         Args:
             note_name (str): The name for your new note (without the .md
                 extension). This name must be unique.
-            content (str, optional): The initial content to write in the note.
-                If not provided, an empty note will be created. Defaults to "".
+            content (str): The initial content to write in the note.
 
         Returns:
             str: A message confirming the creation of the note or an error if
-                 the note name is not valid or already exists.
+                the note name is not valid or already exists.
         """
         try:
             note_path = self.working_directory / f"{note_name}.md"

camel/toolkits/search_toolkit.py

@@ -13,6 +13,7 @@
 # ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========
 import os
 from typing import Any, Dict, List, Literal, Optional, TypeAlias, Union, cast
+from urllib.parse import quote
 
 import requests
 
@@ -35,24 +36,19 @@ class SearchToolkit(BaseToolkit):
     def __init__(
         self,
         timeout: Optional[float] = None,
-        number_of_result_pages: int = 10,
         exclude_domains: Optional[List[str]] = None,
     ):
-        r"""Initializes the RedditToolkit with the specified number of retries
-        and delay.
+        r"""Initializes the SearchToolkit.
 
         Args:
             timeout (float): Timeout for API requests in seconds.
                 (default: :obj:`None`)
-            number_of_result_pages (int): The number of result pages to
-                retrieve. (default: :obj:`10`)
             exclude_domains (Optional[List[str]]): List of domains to
                 exclude from search results. Currently only supported
                 by the `search_google` function.
                 (default: :obj:`None`)
         """
         super().__init__(timeout=timeout)
-        self.number_of_result_pages = number_of_result_pages
         self.exclude_domains = exclude_domains
 
     @dependencies_required("wikipedia")
@@ -167,7 +163,10 @@ class SearchToolkit(BaseToolkit):
 
     @dependencies_required("duckduckgo_search")
     def search_duckduckgo(
-        self, query: str, source: str = "text"
+        self,
+        query: str,
+        source: str = "text",
+        number_of_result_pages: int = 10,
     ) -> List[Dict[str, Any]]:
         r"""Use DuckDuckGo search engine to search information for
         the given query.
@@ -180,6 +179,10 @@ class SearchToolkit(BaseToolkit):
             query (str): The query to be searched.
             source (str): The type of information to query (e.g., "text",
                 "images", "videos"). Defaults to "text".
+            number_of_result_pages (int): The number of result pages to
+                retrieve. Adjust this based on your task - use fewer results
+                for focused searches and more for comprehensive searches.
+                (default: :obj:`10`)
 
         Returns:
             List[Dict[str, Any]]: A list of dictionaries where each dictionary
@@ -194,7 +197,7 @@ class SearchToolkit(BaseToolkit):
         if source == "text":
             try:
                 results = ddgs.text(
-                    keywords=query, max_results=self.number_of_result_pages
+                    keywords=query, max_results=number_of_result_pages
                 )
             except RequestException as e:
                 # Handle specific exceptions or general request exceptions
@@ -214,7 +217,7 @@ class SearchToolkit(BaseToolkit):
         elif source == "images":
             try:
                 results = ddgs.images(
-                    keywords=query, max_results=self.number_of_result_pages
+                    keywords=query, max_results=number_of_result_pages
                 )
             except RequestException as e:
                 # Handle specific exceptions or general request exceptions
@@ -235,7 +238,7 @@ class SearchToolkit(BaseToolkit):
         elif source == "videos":
             try:
                 results = ddgs.videos(
-                    keywords=query, max_results=self.number_of_result_pages
+                    keywords=query, max_results=number_of_result_pages
                 )
             except RequestException as e:
                 # Handle specific exceptions or general request exceptions
@@ -279,6 +282,7 @@ class SearchToolkit(BaseToolkit):
         units: Optional[str] = None,
         extra_snippets: Optional[bool] = None,
         summary: Optional[bool] = None,
+        number_of_result_pages: int = 10,
     ) -> Dict[str, Any]:
         r"""This function queries the Brave search engine API and returns a
         dictionary, representing a search result.
@@ -365,6 +369,10 @@ class SearchToolkit(BaseToolkit):
             summary (Optional[bool]): This parameter enables summary key
                 generation in web search results. This is required for
                 summarizer to be enabled.
+            number_of_result_pages (int): The number of result pages to
+                retrieve. Adjust this based on your task - use fewer results
+                for focused searches and more for comprehensive searches.
+                (default: :obj:`10`)
 
         Returns:
             Dict[str, Any]: A dictionary representing a search result.
@@ -391,7 +399,7 @@ class SearchToolkit(BaseToolkit):
             "country": country,
             "search_lang": search_lang,
             "ui_lang": ui_lang,
-            "count": self.number_of_result_pages,
+            "count": number_of_result_pages,
             "offset": offset,
             "safesearch": safesearch,
             "freshness": freshness,
@@ -444,6 +452,8 @@ class SearchToolkit(BaseToolkit):
         self,
         query: str,
         search_type: str = "web",
+        number_of_result_pages: int = 10,
+        start_page: int = 1,
     ) -> List[Dict[str, Any]]:
         r"""Use Google search engine to search information for the given query.
 
@@ -451,6 +461,15 @@ class SearchToolkit(BaseToolkit):
             query (str): The query to be searched.
             search_type (str): The type of search to perform. Either "web" for
                 web pages or "image" for image search. (default: "web")
+            number_of_result_pages (int): The number of result pages to
+                retrieve. Adjust this based on your task - use fewer results
+                for focused searches and more for comprehensive searches.
+                (default: :obj:`10`)
+            start_page (int): The result page to start from. Use this for
+                pagination - e.g., start_page=1 for results 1-10,
+                start_page=11 for results 11-20, etc. This allows agents to
+                check initial results and continue searching if needed.
+                (default: :obj:`1`)
 
         Returns:
             List[Dict[str, Any]]: A list of dictionaries where each dictionary
@@ -498,13 +517,36 @@ class SearchToolkit(BaseToolkit):
         """
         import requests
 
+        # Validate input parameters
+        if not isinstance(start_page, int) or start_page < 1:
+            raise ValueError("start_page must be a positive integer")
+
+        if (
+            not isinstance(number_of_result_pages, int)
+            or number_of_result_pages < 1
+        ):
+            raise ValueError(
+                "number_of_result_pages must be a positive integer"
+            )
+
+        # Google Custom Search API has a limit of 10 results per request
+        if number_of_result_pages > 10:
+            logger.warning(
+                f"Google API limits results to 10 per request. "
+                f"Requested {number_of_result_pages}, using 10 instead."
+            )
+            number_of_result_pages = 10
+
+        if search_type not in ["web", "image"]:
+            raise ValueError("search_type must be either 'web' or 'image'")
+
         # https://developers.google.com/custom-search/v1/overview
         GOOGLE_API_KEY = os.getenv("GOOGLE_API_KEY")
         # https://cse.google.com/cse/all
         SEARCH_ENGINE_ID = os.getenv("SEARCH_ENGINE_ID")
 
-        # Using the first page
-        start_page_idx = 1
+        # Using the specified start page
+        start_page_idx = start_page
         # Different language may get different result
         search_language = "en"
 
@@ -517,12 +559,14 @@ class SearchToolkit(BaseToolkit):
             modified_query = f"{query} {exclusion_terms}"
             logger.debug(f"Excluded domains, modified query: {modified_query}")
 
+        encoded_query = quote(modified_query)
+
         # Constructing the URL
         # Doc: https://developers.google.com/custom-search/v1/using_rest
         base_url = (
             f"https://www.googleapis.com/customsearch/v1?"
-            f"key={GOOGLE_API_KEY}&cx={SEARCH_ENGINE_ID}&q={modified_query}&start="
-            f"{start_page_idx}&lr={search_language}&num={self.number_of_result_pages}"
+            f"key={GOOGLE_API_KEY}&cx={SEARCH_ENGINE_ID}&q={encoded_query}&start="
+            f"{start_page_idx}&lr={search_language}&num={number_of_result_pages}"
         )
 
         # Add searchType parameter for image search
@@ -534,8 +578,8 @@ class SearchToolkit(BaseToolkit):
         responses = []
         # Fetch the results given the URL
         try:
-            # Make the get
-            result = requests.get(url)
+            result = requests.get(url, timeout=30)
+            result.raise_for_status()  # Raise exception for bad status codes
             data = result.json()
 
             # Get the result items
@@ -574,28 +618,31 @@ class SearchToolkit(BaseToolkit):
 
                         responses.append(response)
                     else:
-                        # Process web search results (existing logic)
-                        # Check metatags are present
-                        if "pagemap" not in search_item:
-                            continue
-                        if "metatags" not in search_item["pagemap"]:
-                            continue
+                        title = search_item.get("title", "")
+                        snippet = search_item.get("snippet", "")
+                        link = search_item.get("link", "")
+
+                        long_description = "N/A"
                         if (
-                            "og:description"
-                            in search_item["pagemap"]["metatags"][0]
+                            "pagemap" in search_item
+                            and "metatags" in search_item["pagemap"]
                         ):
-                            long_description = search_item["pagemap"][
-                                "metatags"
-                            ][0]["og:description"]
-                        else:
-                            long_description = "N/A"
-                        # Get the page title
-                        title = search_item.get("title")
-                        # Page snippet
-                        snippet = search_item.get("snippet")
+                            metatags = search_item["pagemap"]["metatags"]
+                            if metatags and len(metatags) > 0:
+                                meta = metatags[0]
+                                long_description = (
+                                    meta.get("og:description")
+                                    or meta.get("description")
+                                    or meta.get("twitter:description")
+                                    or "N/A"
+                                )
+
+                        if not title and not snippet and not link:
+                            logger.debug(
+                                f"Skipping result {i} due to missing essential fields"
+                            )
+                            continue
 
-                        # Extract the page url
-                        link = search_item.get("link")
                         response = {
                             "result_id": i,
                             "title": title,
@@ -605,26 +652,87 @@ class SearchToolkit(BaseToolkit):
                         }
                         responses.append(response)
             else:
+                logger.debug(f"Google API response without items: {data}")
                 error_info = data.get("error", {})
-                logger.error(
-                    f"Google search failed - API response: {error_info}"
-                )
+                if error_info:
+                    error_code = error_info.get("code", "Unknown")
+                    error_message = error_info.get("message", "Unknown error")
+                    error_details = f"Error {error_code}: {error_message}"
+                else:
+                    error_details = (
+                        "No results found or API returned empty response"
+                    )
+
+                logger.error(f"Google search failed - {error_details}")
                 responses.append(
-                    {
-                        "error": f"Google search failed - "
-                        f"API response: {error_info}"
-                    }
+                    {"error": f"Google search failed - {error_details}"}
                 )
 
+        except requests.exceptions.Timeout:
+            logger.error("Google search request timed out")
+            responses.append(
+                {"error": "Google search request timed out after 30 seconds"}
+            )
+        except requests.exceptions.HTTPError as e:
+            error_message = f"Google API HTTP error: {e}"
+            try:
+                error_data = e.response.json()
+                if "error" in error_data and "message" in error_data["error"]:
+                    api_error_message = error_data["error"]["message"]
+                    error_code = error_data["error"].get(
+                        "code", e.response.status_code
+                    )
+                    error_message = (
+                        f"Google API error {error_code}: {api_error_message}"
+                    )
+
+                    if e.response.status_code == 429:
+                        error_message = f"Google API rate limit exceeded: {api_error_message}"
+                    elif e.response.status_code == 400:
+                        if "API key" in api_error_message:
+                            error_message = (
+                                f"Invalid Google API key: {api_error_message}"
+                            )
+                        else:
+                            error_message = f"Bad request to Google API: {api_error_message}"
+            except (ValueError, KeyError):
+                if e.response.status_code == 429:
+                    error_message = "Google API rate limit exceeded. Please try again later."
+                elif e.response.status_code == 400:
+                    error_message = "Google API bad request. Please check your API key and search engine ID."
+                elif e.response.status_code == 403:
+                    error_message = "Google API access forbidden. Please check your API key permissions."
+
+            logger.error(error_message)
+            responses.append({"error": error_message})
+        except requests.exceptions.RequestException as e:
+            logger.error(f"Google search network error: {e}")
+            responses.append(
+                {"error": f"Network error during Google search: {e!s}"}
+            )
+        except ValueError as e:
+            logger.error(f"Google search response parsing error: {e}")
+            responses.append(
+                {"error": f"Failed to parse Google search response: {e!s}"}
+            )
         except Exception as e:
-            responses.append({"error": f"google search failed: {e!s}"})
+            logger.error(f"Unexpected error during Google search: {e}")
+            responses.append(
+                {"error": f"Unexpected error during Google search: {e!s}"}
+            )
         return responses
 
-    def tavily_search(self, query: str, **kwargs) -> List[Dict[str, Any]]:
+    def tavily_search(
+        self, query: str, number_of_result_pages: int = 10, **kwargs
+    ) -> List[Dict[str, Any]]:
         r"""Use Tavily Search API to search information for the given query.
 
         Args:
             query (str): The query to be searched.
+            number_of_result_pages (int): The number of result pages to
+                retrieve. Adjust this based on your task - use fewer results
+                for focused searches and more for comprehensive searches.
+                (default: :obj:`10`)
             **kwargs: Additional optional parameters supported by Tavily's API:
                 - search_depth (str): "basic" or "advanced" search depth.
                 - topic (str): The search category, e.g., "general" or "news."
@@ -661,7 +769,7 @@ class SearchToolkit(BaseToolkit):
 
         try:
             results = client.search(
-                query, max_results=self.number_of_result_pages, **kwargs
+                query, max_results=number_of_result_pages, **kwargs
             )
             return results
         except Exception as e:
@@ -674,6 +782,7 @@ class SearchToolkit(BaseToolkit):
         freshness: str = "noLimit",
         summary: bool = False,
         page: int = 1,
+        number_of_result_pages: int = 10,
     ) -> Dict[str, Any]:
         r"""Query the Bocha AI search API and return search results.
 
@@ -689,6 +798,10 @@ class SearchToolkit(BaseToolkit):
             summary (bool): Whether to include text summaries in results.
                 Default is False.
             page (int): Page number of results. Default is 1.
+            number_of_result_pages (int): The number of result pages to
+                retrieve. Adjust this based on your task - use fewer results
+                for focused searches and more for comprehensive searches.
+                (default: :obj:`10`)
 
         Returns:
             Dict[str, Any]: A dictionary containing search results, including
@@ -710,7 +823,7 @@ class SearchToolkit(BaseToolkit):
                 "query": query,
                 "freshness": freshness,
                 "summary": summary,
-                "count": self.number_of_result_pages,
+                "count": number_of_result_pages,
                 "page": page,
             },
             ensure_ascii=False,
@@ -728,13 +841,19 @@ class SearchToolkit(BaseToolkit):
         except requests.exceptions.RequestException as e:
             return {"error": f"Bocha AI search failed: {e!s}"}
 
-    def search_baidu(self, query: str) -> Dict[str, Any]:
+    def search_baidu(
+        self, query: str, number_of_result_pages: int = 10
+    ) -> Dict[str, Any]:
         r"""Search Baidu using web scraping to retrieve relevant search
         results. This method queries Baidu's search engine and extracts search
         results including titles, descriptions, and URLs.
 
         Args:
             query (str): Search query string to submit to Baidu.
+            number_of_result_pages (int): The number of result pages to
+                retrieve. Adjust this based on your task - use fewer results
+                for focused searches and more for comprehensive searches.
+                (default: :obj:`10`)
 
         Returns:
             Dict[str, Any]: A dictionary containing search results or error
@@ -752,7 +871,7 @@ class SearchToolkit(BaseToolkit):
                 ),
                 "Referer": "https://www.baidu.com",
             }
-            params = {"wd": query, "rn": str(self.number_of_result_pages)}
+            params = {"wd": query, "rn": str(number_of_result_pages)}
 
             response = requests.get(url, headers=headers, params=params)
             response.encoding = "utf-8"
@@ -781,7 +900,7 @@ class SearchToolkit(BaseToolkit):
                         "url": link,
                     }
                 )
-                if len(results) >= self.number_of_result_pages:
+                if len(results) >= number_of_result_pages:
                     break
 
             if not results:
@@ -795,7 +914,9 @@ class SearchToolkit(BaseToolkit):
         except Exception as e:
             return {"error": f"Baidu scraping error: {e!s}"}
 
-    def search_bing(self, query: str) -> Dict[str, Any]:
+    def search_bing(
+        self, query: str, number_of_result_pages: int = 10
+    ) -> Dict[str, Any]:
         r"""Use Bing search engine to search information for the given query.
 
         This function queries the Chinese version of Bing search engine (cn.
@@ -807,6 +928,10 @@ class SearchToolkit(BaseToolkit):
         Args:
             query (str): The search query string to submit to Bing. Works best
                 with Chinese queries or when Chinese results are preferred.
+            number_of_result_pages (int): The number of result pages to
+                retrieve. Adjust this based on your task - use fewer results
+                for focused searches and more for comprehensive searches.
+                (default: :obj:`10`)
 
         Returns:
             Dict ([str, Any]): A dictionary containing either:
@@ -856,9 +981,7 @@ class SearchToolkit(BaseToolkit):
             result_items = b_results_tag.find_all("li")
 
             results: List[Dict[str, Any]] = []
-            for i in range(
-                min(len(result_items), self.number_of_result_pages)
-            ):
+            for i in range(min(len(result_items), number_of_result_pages)):
                 row = result_items[i]
                 if not isinstance(row, Tag):
                     continue
@@ -927,6 +1050,7 @@ class SearchToolkit(BaseToolkit):
         exclude_text: Optional[List[str]] = None,
         use_autoprompt: bool = True,
         text: bool = False,
+        number_of_result_pages: int = 10,
     ) -> Dict[str, Any]:
         r"""Use Exa search API to perform intelligent web search with optional
         content extraction.
@@ -948,6 +1072,10 @@ class SearchToolkit(BaseToolkit):
                 enhance the query. (default: :obj:`True`)
             text (bool): Whether to include webpage contents in results.
                 (default: :obj:`False`)
+            number_of_result_pages (int): The number of result pages to
+                retrieve. Must be between 1 and 100. Adjust this based on
+                your task - use fewer results for focused searches and more
+                for comprehensive searches. (default: :obj:`10`)
 
         Returns:
             Dict[str, Any]: A dict containing search results and metadata:
@@ -967,8 +1095,8 @@ class SearchToolkit(BaseToolkit):
             exa = Exa(EXA_API_KEY)
 
             if (
-                self.number_of_result_pages is not None
-                and not 0 < self.number_of_result_pages <= 100
+                number_of_result_pages is not None
+                and not 0 < number_of_result_pages <= 100
             ):
                 raise ValueError("num_results must be between 1 and 100")
 
@@ -996,7 +1124,7 @@ class SearchToolkit(BaseToolkit):
                         query=query,
                         type=search_type,
                         category=category,
-                        num_results=self.number_of_result_pages,
+                        num_results=number_of_result_pages,
                         include_text=include_text,
                         exclude_text=exclude_text,
                         use_autoprompt=use_autoprompt,
@@ -1010,7 +1138,7 @@ class SearchToolkit(BaseToolkit):
                         query=query,
                         type=search_type,
                         category=category,
-                        num_results=self.number_of_result_pages,
+                        num_results=number_of_result_pages,
                         include_text=include_text,
                         exclude_text=exclude_text,
                         use_autoprompt=use_autoprompt,
@@ -1043,6 +1171,7 @@ class SearchToolkit(BaseToolkit):
         return_main_text: bool = False,
         return_markdown_text: bool = True,
         enable_rerank: bool = True,
+        number_of_result_pages: int = 10,
     ) -> Dict[str, Any]:
         r"""Query the Alibaba Tongxiao search API and return search results.
 
@@ -1071,6 +1200,10 @@ class SearchToolkit(BaseToolkit):
             enable_rerank (bool): Whether to enable result reranking. If
                 response time is critical, setting this to False can reduce
                 response time by approximately 140ms. (default: :obj:`True`)
+            number_of_result_pages (int): The number of result pages to
+                retrieve. Adjust this based on your task - use fewer results
+                for focused searches and more for comprehensive searches.
+                (default: :obj:`10`)
 
         Returns:
             Dict[str, Any]: A dictionary containing either search results with
@@ -1096,7 +1229,7 @@ class SearchToolkit(BaseToolkit):
         params: Dict[str, Union[str, int]] = {
             "query": query,
             "timeRange": time_range,
-            "page": self.number_of_result_pages,
+            "page": number_of_result_pages,
             "returnMainText": str(return_main_text).lower(),
             "returnMarkdownText": str(return_markdown_text).lower(),
             "enableRerank": str(enable_rerank).lower(),