agno 0.1.2__py3-none-any.whl → 2.3.13__py3-none-any.whl

agno/tools/scrapegraph.py

@@ -1,62 +1,222 @@
 import json
-import os
-from typing import Optional
+from os import getenv
+from typing import Any, List, Optional
 
 from agno.tools import Toolkit
+from agno.utils.log import log_debug, log_error
 
 try:
     from scrapegraph_py import Client
+    from scrapegraph_py.logger import sgai_logger
 except ImportError:
     raise ImportError("`scrapegraph-py` not installed. Please install using `pip install scrapegraph-py`")
 
+# Set logging level
+sgai_logger.set_logging(level="INFO")
+
 
 class ScrapeGraphTools(Toolkit):
     def __init__(
         self,
         api_key: Optional[str] = None,
-        smartscraper: bool = True,
-        markdownify: bool = False,
+        enable_smartscraper: bool = True,
+        enable_markdownify: bool = False,
+        enable_crawl: bool = False,
+        enable_searchscraper: bool = False,
+        enable_agentic_crawler: bool = False,
+        enable_scrape: bool = False,
+        render_heavy_js: bool = False,
+        all: bool = False,
+        **kwargs,
     ):
-        super().__init__(name="scrapegraph_tools")
-
-        self.api_key: Optional[str] = api_key or os.getenv("SGAI_API_KEY")
+        self.api_key: Optional[str] = api_key or getenv("SGAI_API_KEY")
         self.client = Client(api_key=self.api_key)
+        self.render_heavy_js = render_heavy_js
 
         # Start with smartscraper by default
         # Only enable markdownify if smartscraper is False
-        if not smartscraper:
-            markdownify = True
+        if not enable_smartscraper and not all:
+            enable_markdownify = True
+
+        tools: List[Any] = []
+        if enable_smartscraper or all:
+            tools.append(self.smartscraper)
+        if enable_markdownify or all:
+            tools.append(self.markdownify)
+        if enable_crawl or all:
+            tools.append(self.crawl)
+        if enable_searchscraper or all:
+            tools.append(self.searchscraper)
+        if enable_agentic_crawler or all:
+            tools.append(self.agentic_crawler)
+        if enable_scrape or all:
+            tools.append(self.scrape)
 
-        if smartscraper:
-            self.register(self.smartscraper)
-        if markdownify:
-            self.register(self.markdownify)
+        super().__init__(name="scrapegraph_tools", tools=tools, **kwargs)
 
     def smartscraper(self, url: str, prompt: str) -> str:
-        """Use this function to extract structured data from a webpage using LLM.
+        """Extract structured data from a webpage using LLM.
         Args:
             url (str): The URL to scrape
             prompt (str): Natural language prompt describing what to extract
         Returns:
             The structured data extracted from the webpage
         """
-
         try:
+            log_debug(f"ScrapeGraph smartscraper request for URL: {url}")
             response = self.client.smartscraper(website_url=url, user_prompt=prompt)
             return json.dumps(response["result"])
         except Exception as e:
-            return json.dumps({"error": str(e)})
+            error_msg = f"Smartscraper failed: {str(e)}"
+            log_error(error_msg)
+            return f"Error: {error_msg}"
 
     def markdownify(self, url: str) -> str:
-        """Use this function to convert a webpage to markdown format.
+        """Convert a webpage to markdown format.
         Args:
             url (str): The URL to convert
         Returns:
             The markdown version of the webpage
         """
-
         try:
+            log_debug(f"ScrapeGraph markdownify request for URL: {url}")
             response = self.client.markdownify(website_url=url)
             return response["result"]
         except Exception as e:
-            return f"Error converting to markdown: {str(e)}"
+            error_msg = f"Markdownify failed: {str(e)}"
+            log_error(error_msg)
+            return f"Error: {error_msg}"
+
+    def crawl(
+        self,
+        url: str,
+        prompt: str,
+        schema: dict,
+        cache_website: bool = True,
+        depth: int = 2,
+        max_pages: int = 2,
+        same_domain_only: bool = True,
+        batch_size: int = 1,
+    ) -> str:
+        """Crawl a website and extract structured data
+        Args:
+            url (str): The URL to crawl
+            prompt (str): Natural language prompt describing what to extract
+            schema (dict): JSON schema for extraction
+            cache_website (bool): Whether to cache the website
+            depth (int): Crawl depth
+            max_pages (int): Max number of pages to crawl
+            same_domain_only (bool): Restrict to same domain
+            batch_size (int): Batch size for crawling
+        Returns:
+            The structured data extracted from the website
+        """
+        try:
+            log_debug(f"ScrapeGraph crawl request for URL: {url}")
+            response = self.client.crawl(
+                url=url,
+                prompt=prompt,
+                data_schema=schema,
+                cache_website=cache_website,
+                depth=depth,
+                max_pages=max_pages,
+                same_domain_only=same_domain_only,
+                batch_size=batch_size,
+            )
+            return json.dumps(response, indent=2)
+        except Exception as e:
+            error_msg = f"Crawl failed: {str(e)}"
+            log_error(error_msg)
+            return f"Error: {error_msg}"
+
+    def agentic_crawler(
+        self,
+        url: str,
+        steps: List[str],
+        use_session: bool = True,
+        user_prompt: Optional[str] = None,
+        output_schema: Optional[dict] = None,
+        ai_extraction: bool = False,
+    ) -> str:
+        """Perform agentic crawling with automated browser actions and optional AI extraction.
+
+        This tool can:
+        1. Navigate to a website
+        2. Perform a series of automated actions (like filling forms, clicking buttons)
+        3. Extract the resulting HTML content as markdown
+        4. Optionally use AI to extract structured data
+
+        Args:
+            url (str): The URL to scrape
+            steps (List[str]): List of steps to perform on the webpage (e.g., ["Type email in input box", "click login"])
+            use_session (bool): Whether to use session for the scraping (default: True)
+            user_prompt (Optional[str]): Prompt for AI extraction (only used when ai_extraction=True)
+            output_schema (Optional[dict]): Schema for structured data extraction (only used when ai_extraction=True)
+            ai_extraction (bool): Whether to use AI for data extraction from the scraped content (default: False)
+
+        Returns:
+            JSON string containing the scraping results, including request_id, status, and extracted data
+        """
+        try:
+            log_debug(f"ScrapeGraph agentic_crawler request for URL: {url}")
+
+            # Prepare parameters for the API call
+            params = {"url": url, "steps": steps, "use_session": use_session, "ai_extraction": ai_extraction}
+
+            # Add optional parameters only if they are provided
+            if user_prompt:
+                params["user_prompt"] = user_prompt
+            if output_schema:
+                params["output_schema"] = output_schema
+
+            # Call the agentic scraper API
+            response = self.client.agenticscraper(**params)
+            return json.dumps(response, indent=2)
+
+        except Exception as e:
+            error_msg = f"Agentic crawler failed: {str(e)}"
+            log_error(error_msg)
+            return f"Error: {error_msg}"
+
+    def searchscraper(self, user_prompt: str) -> str:
+        """Search the web and extract information from the web.
+        Args:
+            user_prompt (str): Search query
+        Returns:
+            JSON of the search results
+        """
+        try:
+            log_debug(f"ScrapeGraph searchscraper request with prompt: {user_prompt}")
+            response = self.client.searchscraper(user_prompt=user_prompt)
+            return json.dumps(response["result"])
+        except Exception as e:
+            error_msg = f"Searchscraper failed: {str(e)}"
+            log_error(error_msg)
+            return f"Error: {error_msg}"
+
+    def scrape(
+        self,
+        website_url: str,
+        headers: Optional[dict] = None,
+    ) -> str:
+        """Get raw HTML content from a website using the ScrapeGraphAI scrape API.
+
+        Args:
+            website_url (str): The URL of the website to scrape
+            headers (Optional[dict]): Optional headers to send with the request
+
+        Returns:
+            JSON string containing the HTML content and metadata
+        """
+        try:
+            log_debug(f"ScrapeGraph scrape request for URL: {website_url}")
+            response = self.client.scrape(
+                website_url=website_url,
+                headers=headers,
+                render_heavy_js=self.render_heavy_js,
+            )
+            return json.dumps(response, indent=2)
+        except Exception as e:
+            error_msg = f"Scrape failed: {str(e)}"
+            log_error(error_msg)
+            return f"Error: {error_msg}"

agno/tools/searxng.py

@@ -1,11 +1,11 @@
 import json
 import urllib.parse
-from typing import List, Optional
+from typing import Any, List, Optional
 
 import httpx
 
-from agno.tools.toolkit import Toolkit
-from agno.utils.log import logger
+from agno.tools import Toolkit
+from agno.utils.log import log_info
 
 
 class Searxng(Toolkit):
@@ -14,38 +14,26 @@ class Searxng(Toolkit):
         host: str,
         engines: List[str] = [],
         fixed_max_results: Optional[int] = None,
-        images: bool = False,
-        it: bool = False,
-        map: bool = False,
-        music: bool = False,
-        news: bool = False,
-        science: bool = False,
-        videos: bool = False,
+        **kwargs,
     ):
-        super().__init__(name="searxng")
-
         self.host = host
         self.engines = engines
         self.fixed_max_results = fixed_max_results
 
-        self.register(self.search)
-
-        if images:
-            self.register(self.image_search)
-        if it:
-            self.register(self.it_search)
-        if map:
-            self.register(self.map_search)
-        if music:
-            self.register(self.music_search)
-        if news:
-            self.register(self.news_search)
-        if science:
-            self.register(self.science_search)
-        if videos:
-            self.register(self.video_search)
-
-    def search(self, query: str, max_results: int = 5) -> str:
+        tools: List[Any] = [
+            self.search_web,
+            self.image_search,
+            self.it_search,
+            self.map_search,
+            self.music_search,
+            self.news_search,
+            self.science_search,
+            self.video_search,
+        ]
+
+        super().__init__(name="searxng", tools=tools, **kwargs)
+
+    def search_web(self, query: str, max_results: int = 5) -> str:
         """Use this function to search the web.
 
         Args:
@@ -150,7 +138,7 @@ class Searxng(Toolkit):
         if category:
             url += f"&categories={category}"
 
-        logger.info(f"Fetching results from searxng: {url}")
+        log_info(f"Fetching results from searxng: {url}")
         try:
             resp = httpx.get(url).json()
             results = self.fixed_max_results or max_results
@@ -158,3 +146,7 @@ class Searxng(Toolkit):
             return json.dumps(resp)
         except Exception as e:
             return f"Error fetching results from searxng: {e}"
+
+
+# Alias for consistency with other tools
+SearxngTools = Searxng

agno/tools/serpapi.py

@@ -1,9 +1,9 @@
 import json
 from os import getenv
-from typing import Optional
+from typing import Any, List, Optional
 
 from agno.tools import Toolkit
-from agno.utils.log import logger
+from agno.utils.log import log_info, logger
 
 try:
     import serpapi
@@ -15,17 +15,22 @@ class SerpApiTools(Toolkit):
     def __init__(
         self,
         api_key: Optional[str] = None,
-        search_youtube: bool = False,
+        enable_search_google: bool = True,
+        enable_search_youtube: bool = False,
+        all: bool = False,
+        **kwargs,
     ):
-        super().__init__(name="serpapi_tools")
-
         self.api_key = api_key or getenv("SERP_API_KEY")
         if not self.api_key:
             logger.warning("No Serpapi API key provided")
 
-        self.register(self.search_google)
-        if search_youtube:
-            self.register(self.search_youtube)
+        tools: List[Any] = []
+        if all or enable_search_google:
+            tools.append(self.search_google)
+        if all or enable_search_youtube:
+            tools.append(self.search_youtube)
+
+        super().__init__(name="serpapi_tools", tools=tools, **kwargs)
 
     def search_google(self, query: str, num_results: int = 10) -> str:
         """
@@ -51,7 +56,7 @@ class SerpApiTools(Toolkit):
             if not query:
                 return "Please provide a query to search for"
 
-            logger.info(f"Searching Google for: {query}")
+            log_info(f"Searching Google for: {query}")
 
             params = {"q": query, "api_key": self.api_key, "num": num_results}
 
@@ -92,7 +97,7 @@ class SerpApiTools(Toolkit):
             if not query:
                 return "Please provide a query to search for"
 
-            logger.info(f"Searching Youtube for: {query}")
+            log_info(f"Searching Youtube for: {query}")
 
             params = {"search_query": query, "api_key": self.api_key}
 

agno/tools/serper.py

@@ -0,0 +1,255 @@
+import json
+from os import getenv
+from typing import Any, Dict, List, Optional
+
+import requests
+
+from agno.tools import Toolkit
+from agno.utils.log import log_debug, log_error, log_warning
+
+
+class SerperTools(Toolkit):
+    def __init__(
+        self,
+        api_key: Optional[str] = None,
+        location: str = "us",
+        language: str = "en",
+        num_results: int = 10,
+        date_range: Optional[str] = None,
+        enable_search: bool = True,
+        enable_search_news: bool = True,
+        enable_search_scholar: bool = True,
+        enable_scrape_webpage: bool = True,
+        all: bool = False,
+        **kwargs,
+    ):
+        """
+        Initialize the SerperTools.
+
+        Args:
+            api_key Optional[str]: The Serper API key.
+            location Optional[str]: The Google location code for search results.
+            language Optional[str]: The language code for search results.
+            num_results Optional[int]: The number of search results to retrieve.
+            date_range Optional[str]: Default date range filter for searches.
+        """
+        self.api_key = api_key or getenv("SERPER_API_KEY")
+        if not self.api_key:
+            log_debug("No Serper API key provided")
+
+        self.location = location
+        self.language = language
+        self.num_results = num_results
+        self.date_range = date_range
+
+        tools: List[Any] = []
+        if all or enable_search:
+            tools.append(self.search_web)
+        if all or enable_search_news:
+            tools.append(self.search_news)
+        if all or enable_search_scholar:
+            tools.append(self.search_scholar)
+        if all or enable_scrape_webpage:
+            tools.append(self.scrape_webpage)
+
+        super().__init__(name="serper_tools", tools=tools, **kwargs)
+
+    def _make_request(self, endpoint: str, params: Dict[str, Any]) -> Dict[str, Any]:
+        """
+        Makes a request to the Serper API.
+
+        Args:
+            endpoint (str): The API endpoint
+            params (Dict[str, Any]): Request parameters
+
+        Returns:
+            Dict[str, Any]: Search response
+        """
+        try:
+            if not self.api_key:
+                log_error("No Serper API key provided")
+                return {"success": False, "error": "Please provide a Serper API key"}
+
+            url = f"https://google.serper.dev/{endpoint}"
+            if endpoint == "scrape":
+                url = "https://scrape.serper.dev"
+
+            headers = {"X-API-KEY": self.api_key, "Content-Type": "application/json"}
+
+            # Add optional parameters
+            if self.date_range:
+                params["tbs"] = self.date_range
+            if self.location:
+                params["gl"] = self.location
+
+            if self.language:
+                params["hl"] = self.language
+
+            payload = json.dumps(params)
+
+            log_debug(f"Making request to {url} with params: {params}")
+            response = requests.request("POST", url, headers=headers, data=payload)
+            response.raise_for_status()
+
+            log_debug(f"Successfully received response from {endpoint} endpoint")
+            return {"success": True, "data": response.json(), "raw_response": response.text}
+        except Exception as e:
+            log_error(f"Serper API error: {str(e)}")
+            return {"success": False, "error": str(e)}
+
+    def search_web(
+        self,
+        query: str,
+        num_results: Optional[int] = None,
+    ) -> str:
+        """
+        Searches Google for the provided query using the Serper API.
+
+        Args:
+            query (str): The search query to search for on Google.
+            num_results (int, optional): Number of search results to retrieve.
+
+        Returns:
+            str: A JSON-formatted string containing the search results or an error message if the search fails.
+        """
+        try:
+            if not query:
+                return json.dumps({"error": "Please provide a query to search for"}, indent=2)
+
+            log_debug(f"Searching Google for: {query}")
+
+            params = {
+                "q": query,
+                "num": num_results or self.num_results,
+            }
+
+            result = self._make_request("search", params)
+
+            if result["success"]:
+                log_debug(f"Successfully found Google search results for query: {query}")
+                return result["raw_response"]
+            else:
+                log_error(f"Error searching Google for query {query}: {result['error']}")
+                return json.dumps({"error": result["error"]}, indent=2)
+
+        except Exception as e:
+            log_error(f"Unexpected error searching Google for query {query}: {e}")
+            return json.dumps({"error": f"An unexpected error occurred: {str(e)}"}, indent=2)
+
+    def search_news(
+        self,
+        query: str,
+        num_results: Optional[int] = None,
+    ) -> str:
+        """
+        Searches for news articles using the Serper News API.
+
+        Args:
+            query (str): The search query for news articles.
+            num_results (int, optional): Number of news results to retrieve.
+
+        Returns:
+            str: A JSON-formatted string containing the news search results or an error message.
+        """
+        try:
+            if not query:
+                return json.dumps({"error": "Please provide a query to search for news"}, indent=2)
+
+            log_debug(f"Searching news for: {query}")
+
+            params = {
+                "q": query,
+                "num": num_results or self.num_results,
+            }
+
+            result = self._make_request("news", params)
+
+            if result["success"]:
+                log_debug(f"Successfully found {num_results or self.num_results} news articles for query: {query}")
+                return result["raw_response"]
+            else:
+                log_error(f"Error searching news for query {query}: {result['error']}")
+                return json.dumps({"error": result["error"]}, indent=2)
+
+        except Exception as e:
+            log_error(f"Unexpected error searching news for query {query}: {e}")
+            return json.dumps({"error": f"An unexpected error occurred: {str(e)}"}, indent=2)
+
+    def search_scholar(
+        self,
+        query: str,
+        num_results: Optional[int] = None,
+    ) -> str:
+        """
+        Searches for academic papers using Google Scholar via Serper API.
+
+        Args:
+            query (str): The search query for academic papers.
+            num_results (int, optional): Number of academic papers to retrieve.
+
+        Returns:
+            str: A JSON-formatted string containing the scholar search results or an error message.
+        """
+        try:
+            if not query:
+                return json.dumps({"error": "Please provide a query to search for academic papers"}, indent=2)
+
+            log_debug(f"Searching scholar for: {query}")
+
+            params = {
+                "q": query,
+                "num": num_results or self.num_results,
+            }
+
+            result = self._make_request("scholar", params)
+
+            if result["success"]:
+                log_debug(f"Successfully found academic papers for query: {query}")
+                return result["raw_response"]
+            else:
+                log_error(f"Error searching scholar for query {query}: {result['error']}")
+                return json.dumps({"error": result["error"]}, indent=2)
+
+        except Exception as e:
+            log_error(f"Unexpected error searching scholar for query {query}: {e}")
+            return json.dumps({"error": f"An unexpected error occurred: {str(e)}"}, indent=2)
+
+    def scrape_webpage(
+        self,
+        url: str,
+        markdown: bool = False,
+    ) -> str:
+        """
+        Scrapes and extracts content from a webpage using the Serper scraping API.
+
+        Args:
+            url (str): The URL of the webpage to scrape.
+            markdown (bool, optional): Return content in markdown format (default: False).
+
+        Returns:
+            str: A JSON-formatted string containing the scraped webpage content or an error message.
+        """
+        try:
+            if not url:
+                log_warning("No URL provided to scrape")
+                return json.dumps({"error": "Please provide a URL to scrape"}, indent=2)
+
+            log_debug(f"Scraping webpage: {url}")
+
+            params = {
+                "url": url,
+                "includeMarkdown": markdown,
+            }
+
+            result = self._make_request("scrape", params)
+
+            if result["success"]:
+                log_debug(f"Successfully scraped webpage: {url}")
+                return result["raw_response"]
+            else:
+                log_error(f"Error scraping webpage {url}: {result['error']}")
+                return json.dumps({"error": result["error"]}, indent=2)
+
+        except Exception as e:
+            log_error(f"Unexpected error scraping webpage {url}: {e}")
+            return json.dumps({"error": f"An unexpected error occurred: {str(e)}"}, indent=2)