agno 0.1.2__py3-none-any.whl → 2.3.13__py3-none-any.whl

agno/tools/crawl4ai.py

@@ -1,10 +1,11 @@
 import asyncio
-from typing import Optional
+from typing import Any, Dict, List, Optional, Union
 
 from agno.tools import Toolkit
+from agno.utils.log import log_debug, log_warning
 
 try:
-    from crawl4ai import AsyncWebCrawler, CacheMode
+    from crawl4ai import AsyncWebCrawler, BrowserConfig, CrawlerRunConfig
 except ImportError:
     raise ImportError("`crawl4ai` not installed. Please install using `pip install crawl4ai`")
 
@@ -12,51 +13,146 @@ except ImportError:
 class Crawl4aiTools(Toolkit):
     def __init__(
         self,
-        max_length: Optional[int] = 1000,
+        max_length: Optional[int] = 5000,
+        timeout: int = 60,
+        use_pruning: bool = False,
+        pruning_threshold: float = 0.48,
+        bm25_threshold: float = 1.0,
+        headless: bool = True,
+        wait_until: str = "domcontentloaded",
+        enable_crawl: bool = True,
+        all: bool = False,
+        **kwargs,
     ):
-        super().__init__(name="crawl4ai_tools")
+        tools = []
+        if all or enable_crawl:
+            tools.append(self.crawl)
 
+        super().__init__(name="crawl4ai_tools", tools=tools, **kwargs)
         self.max_length = max_length
-
-        self.register(self.web_crawler)
-
-    def web_crawler(self, url: str, max_length: Optional[int] = None) -> str:
-        """
-        Crawls a website using crawl4ai's WebCrawler.
-
-        :param url: The URL to crawl.
-        :param max_length: The maximum length of the result.
-
-        :return: The results of the crawling.
-        """
-        if url is None:
-            return "No URL provided"
-
-        # Run the async crawler function synchronously
-        return asyncio.run(self._async_web_crawler(url, max_length))
-
-    async def _async_web_crawler(self, url: str, max_length: Optional[int] = None) -> str:
+        self.timeout = timeout
+        self.use_pruning = use_pruning
+        self.pruning_threshold = pruning_threshold
+        self.bm25_threshold = bm25_threshold
+        self.wait_until = wait_until
+        self.headless = headless
+
+    def _build_config(self, search_query: Optional[str] = None) -> Dict[str, Any]:
+        """Build CrawlerRunConfig parameters from toolkit settings."""
+        config_params = {
+            "page_timeout": self.timeout * 1000,  # Convert to milliseconds
+            "wait_until": self.wait_until,
+            "cache_mode": "bypass",  # Don't use cache for fresh results
+            "verbose": False,
+        }
+
+        # Handle content filtering
+        if self.use_pruning or search_query:
+            try:
+                from crawl4ai.content_filter_strategy import BM25ContentFilter, PruningContentFilter
+                from crawl4ai.markdown_generation_strategy import DefaultMarkdownGenerator
+
+                if search_query:
+                    # Use BM25 for query-specific extraction
+                    content_filter = BM25ContentFilter(user_query=search_query, bm25_threshold=self.bm25_threshold)
+                    log_debug(f"Using BM25ContentFilter for query: {search_query}")
+                else:
+                    # Use pruning for general cleanup
+                    content_filter = PruningContentFilter(
+                        threshold=self.pruning_threshold, threshold_type="fixed", min_word_threshold=2
+                    )
+                    log_debug("Using PruningContentFilter for general cleanup")
+
+                config_params["markdown_generator"] = DefaultMarkdownGenerator(content_filter=content_filter)
+                log_debug("Using DefaultMarkdownGenerator with content_filter")
+            except ImportError:
+                # If advanced features not available, continue without them
+                log_warning("crawl4ai.content_filter_strategy or crawl4ai.markdown_generation_strategy not installed")
+                pass
+
+        return config_params
+
+    def crawl(self, url: Union[str, List[str]], search_query: Optional[str] = None) -> Union[str, Dict[str, str]]:
         """
-        Asynchronous method to crawl a website using AsyncWebCrawler.
+        Crawl URLs and extract their text content.
 
-        :param url: The URL to crawl.
+        Args:
+            url: Single URL string or list of URLs to crawl
+            search_query: Optional query string to filter content using BM25 algorithm
 
-        :return: The results of the crawling as a markdown string, or None if no result.
+        Returns:
+            The extracted text content from the URL(s)
         """
-
-        async with AsyncWebCrawler(thread_safe=True) as crawler:
-            result = await crawler.arun(url=url, cache_mode=CacheMode.BYPASS)
-
-            # Determine the length to use
-            length = self.max_length or max_length
-            if not result.markdown:
-                return "No result"
-
-            # Remove spaces and truncate if length is specified
-            if length:
-                result = result.markdown[:length]
-                result = result.replace(" ", "")
-                return result
-
-            result = result.markdown.replace(" ", "")
-        return result
+        if not url:
+            return "Error: No URL provided"
+
+        # Handle single URL
+        if isinstance(url, str):
+            return asyncio.run(self._async_crawl(url, search_query))
+
+        # Handle multiple URLs
+        results = {}
+        for single_url in url:
+            results[single_url] = asyncio.run(self._async_crawl(single_url, search_query))
+        return results
+
+    async def _async_crawl(self, url: str, search_query: Optional[str] = None) -> str:
+        """Crawl a single URL and extract content."""
+        try:
+            # Use BrowserConfig to suppress crawl4ai logs
+            browser_config = BrowserConfig(
+                headless=self.headless,
+                verbose=False,
+            )
+
+            async with AsyncWebCrawler(config=browser_config) as crawler:
+                # Build configuration from parameters
+                config_params = self._build_config(search_query)
+
+                config = CrawlerRunConfig(**config_params)
+                log_debug(f"Crawling {url} with config: {config}")
+                result = await crawler.arun(url=url, config=config)
+
+                # Process the result
+                if not result:
+                    return "Error: No content found"
+
+                log_debug(f"Result attributes: {dir(result)}")
+                log_debug(f"Result success: {getattr(result, 'success', 'N/A')}")
+
+                # Try to get markdown content
+                content = ""
+                if hasattr(result, "fit_markdown") and result.fit_markdown:
+                    content = result.fit_markdown
+                    log_debug("Using fit_markdown")
+                elif hasattr(result, "markdown") and result.markdown:
+                    if hasattr(result.markdown, "raw_markdown"):
+                        content = result.markdown.raw_markdown
+                        log_debug("Using markdown.raw_markdown")
+                    else:
+                        content = str(result.markdown)
+                        log_debug("Using str(markdown)")
+                else:
+                    # Try to get any text content
+                    if hasattr(result, "text"):
+                        content = result.text
+                        log_debug("Using text attribute")
+                    elif hasattr(result, "html"):
+                        log_warning("Only HTML available, no markdown extracted")
+                        return "Error: Could not extract markdown from page"
+
+                if not content:
+                    log_warning(f"No content extracted. Result type: {type(result)}")
+                    return "Error: No readable content extracted"
+
+                log_debug(f"Extracted content length: {len(content)}")
+
+                # Truncate if needed
+                if self.max_length and len(content) > self.max_length:
+                    content = content[: self.max_length] + "..."
+
+                return content
+
+        except Exception as e:
+            log_warning(f"Exception during crawl: {str(e)}")
+            return f"Error crawling {url}: {str(e)}"

agno/tools/csv_toolkit.py

@@ -4,7 +4,7 @@ from pathlib import Path
 from typing import Any, Dict, List, Optional, Union
 
 from agno.tools import Toolkit
-from agno.utils.log import logger
+from agno.utils.log import log_debug, log_info, logger
 
 
 class CsvTools(Toolkit):
@@ -12,15 +12,15 @@ class CsvTools(Toolkit):
         self,
         csvs: Optional[List[Union[str, Path]]] = None,
         row_limit: Optional[int] = None,
-        read_csvs: bool = True,
-        list_csvs: bool = True,
-        query_csvs: bool = True,
-        read_column_names: bool = True,
         duckdb_connection: Optional[Any] = None,
         duckdb_kwargs: Optional[Dict[str, Any]] = None,
+        enable_read_csv_file: bool = True,
+        enable_list_csv_files: bool = True,
+        enable_get_columns: bool = True,
+        enable_query_csv_file: bool = True,
+        all: bool = False,
+        **kwargs,
     ):
-        super().__init__(name="csv_tools")
-
         self.csvs: List[Path] = []
         if csvs:
             for _csv in csvs:
@@ -34,18 +34,22 @@ class CsvTools(Toolkit):
         self.duckdb_connection: Optional[Any] = duckdb_connection
         self.duckdb_kwargs: Optional[Dict[str, Any]] = duckdb_kwargs
 
-        if read_csvs:
-            self.register(self.read_csv_file)
-        if list_csvs:
-            self.register(self.list_csv_files)
-        if read_column_names:
-            self.register(self.get_columns)
-        if query_csvs:
+        tools: List[Any] = []
+        if all or enable_read_csv_file:
+            tools.append(self.read_csv_file)
+        if all or enable_list_csv_files:
+            tools.append(self.list_csv_files)
+        if all or enable_get_columns:
+            tools.append(self.get_columns)
+        if all or enable_query_csv_file:
             try:
                 import duckdb  # noqa: F401
+
+                tools.append(self.query_csv_file)
             except ImportError:
-                raise ImportError("`duckdb` not installed. Please install using `pip install duckdb`.")
-            self.register(self.query_csv_file)
+                logger.warning("`duckdb` not installed. Query functionality disabled.")
+
+        super().__init__(name="csv_tools", tools=tools, **kwargs)
 
     def list_csv_files(self) -> str:
         """Returns a list of available csv files
@@ -69,7 +73,7 @@ class CsvTools(Toolkit):
             if csv_name not in [_csv.stem for _csv in self.csvs]:
                 return f"File: {csv_name} not found, please use one of {self.list_csv_files()}"
 
-            logger.info(f"Reading file: {csv_name}")
+            log_info(f"Reading file: {csv_name}")
             file_path = [_csv for _csv in self.csvs if _csv.stem == csv_name][0]
 
             # Read the csv file
@@ -99,7 +103,7 @@ class CsvTools(Toolkit):
             if csv_name not in [_csv.stem for _csv in self.csvs]:
                 return f"File: {csv_name} not found, please use one of {self.list_csv_files()}"
 
-            logger.info(f"Reading columns from file: {csv_name}")
+            log_info(f"Reading columns from file: {csv_name}")
             file_path = [_csv for _csv in self.csvs if _csv.stem == csv_name][0]
 
             # Get the columns of the csv file
@@ -134,7 +138,7 @@ class CsvTools(Toolkit):
                 return f"File: {csv_name} not found, please use one of {self.list_csv_files()}"
 
             # Load the csv file into duckdb
-            logger.info(f"Loading csv file: {csv_name}")
+            log_info(f"Loading csv file: {csv_name}")
             file_path = [_csv for _csv in self.csvs if _csv.stem == csv_name][0]
 
             # Create duckdb connection
@@ -146,7 +150,9 @@ class CsvTools(Toolkit):
                 return "Error connecting to DuckDB, please check the connection."
 
             # Create a table from the csv file
-            con.execute(f"CREATE TABLE {csv_name} AS SELECT * FROM read_csv_auto('{file_path}')")
+            con.execute(
+                f"CREATE TABLE {csv_name} AS SELECT * FROM read_csv('{file_path}', ignore_errors=false, auto_detect=true)"
+            )
 
             # -*- Format the SQL Query
             # Remove backticks
@@ -154,7 +160,7 @@ class CsvTools(Toolkit):
             # If there are multiple statements, only run the first one
             formatted_sql = formatted_sql.split(";")[0]
             # -*- Run the SQL Query
-            logger.info(f"Running query: {formatted_sql}")
+            log_info(f"Running query: {formatted_sql}")
             query_result = con.sql(formatted_sql)
             result_output = "No output"
             if query_result is not None:
@@ -172,7 +178,7 @@ class CsvTools(Toolkit):
                 except AttributeError:
                     result_output = str(query_result)
 
-            logger.debug(f"Query result: {result_output}")
+            log_debug(f"Query result: {result_output}")
             return result_output
         except Exception as e:
             logger.error(f"Error querying csv: {e}")

agno/tools/dalle.py

@@ -1,11 +1,11 @@
 from os import getenv
-from typing import Literal, Optional
+from typing import Any, List, Literal, Optional
 from uuid import uuid4
 
-from agno.agent import Agent
-from agno.media import ImageArtifact
+from agno.media import Image
 from agno.tools import Toolkit
-from agno.utils.log import logger
+from agno.tools.function import ToolResult
+from agno.utils.log import log_debug, logger
 
 try:
     from openai import OpenAI
@@ -23,9 +23,10 @@ class DalleTools(Toolkit):
         quality: Literal["standard", "hd"] = "standard",
         style: Literal["vivid", "natural"] = "vivid",
         api_key: Optional[str] = None,
+        enable_create_image: bool = True,
+        all: bool = False,
+        **kwargs,
     ):
-        super().__init__(name="dalle")
-
         self.model = model
         self.n = n
         self.size = size
@@ -50,27 +51,32 @@ class DalleTools(Toolkit):
         if not self.api_key:
             logger.error("OPENAI_API_KEY not set. Please set the OPENAI_API_KEY environment variable.")
 
-        self.register(self.create_image)
+        tools: List[Any] = []
+        if all or enable_create_image:
+            tools.append(self.create_image)
+
+        super().__init__(name="dalle", tools=tools, **kwargs)
+
         # TODO:
         # - Add support for response_format
         # - Add support for saving images
         # - Add support for editing images
 
-    def create_image(self, agent: Agent, prompt: str) -> str:
+    def create_image(self, prompt: str) -> ToolResult:
         """Use this function to generate an image for a prompt.
 
         Args:
             prompt (str): A text description of the desired image.
 
         Returns:
-            str: str: A message indicating if the image has been generated successfully or an error message.
+            ToolResult: Result containing the message and generated images.
         """
         if not self.api_key:
-            return "Please set the OPENAI_API_KEY"
+            return ToolResult(content="Please set the OPENAI_API_KEY")
 
         try:
             client = OpenAI(api_key=self.api_key)
-            logger.debug(f"Generating image using prompt: {prompt}")
+            log_debug(f"Generating image using prompt: {prompt}")
             response: ImagesResponse = client.images.generate(
                 prompt=prompt,
                 model=self.model,
@@ -79,18 +85,26 @@ class DalleTools(Toolkit):
                 size=self.size,
                 style=self.style,
             )
-            logger.debug("Image generated successfully")
+            log_debug("Image generated successfully")
 
-            # Update the run response with the image URLs
+            generated_images = []
             response_str = ""
-            for img in response.data:
-                agent.add_image(
-                    ImageArtifact(
-                        id=str(uuid4()), url=img.url, original_prompt=prompt, revised_prompt=img.revised_prompt
-                    )
-                )
-                response_str += f"Image has been generated at the URL {img.url}\n"
-            return response_str
+            if response.data:
+                for img in response.data:
+                    if img.url:
+                        image = Image(
+                            id=str(uuid4()),
+                            url=img.url,
+                            original_prompt=prompt,
+                            revised_prompt=img.revised_prompt,
+                        )
+                        generated_images.append(image)
+                        response_str += f"Image has been generated at the URL {img.url}\n"
+
+            return ToolResult(
+                content=response_str or "No images were generated",
+                images=generated_images if generated_images else None,
+            )
         except Exception as e:
             logger.error(f"Failed to generate image: {e}")
-            return f"Error: {e}"
+            return ToolResult(content=f"Error: {e}")