agno 0.1.2__py3-none-any.whl → 2.3.13__py3-none-any.whl

agno/tools/toolkit.py

@@ -1,43 +1,168 @@
 from collections import OrderedDict
-from typing import Any, Callable, Dict
+from typing import Any, Callable, Dict, List, Optional
 
 from agno.tools.function import Function
-from agno.utils.log import logger
+from agno.utils.log import log_debug, log_warning, logger
 
 
 class Toolkit:
-    def __init__(self, name: str = "toolkit"):
+    # Set to True for toolkits that require connection management (e.g., database connections)
+    # When True, the Agent will automatically call connect() before using tools and close() after
+    _requires_connect: bool = False
+
+    def __init__(
+        self,
+        name: str = "toolkit",
+        tools: List[Callable] = [],
+        instructions: Optional[str] = None,
+        add_instructions: bool = False,
+        include_tools: Optional[list[str]] = None,
+        exclude_tools: Optional[list[str]] = None,
+        requires_confirmation_tools: Optional[list[str]] = None,
+        external_execution_required_tools: Optional[list[str]] = None,
+        stop_after_tool_call_tools: Optional[List[str]] = None,
+        show_result_tools: Optional[List[str]] = None,
+        cache_results: bool = False,
+        cache_ttl: int = 3600,
+        cache_dir: Optional[str] = None,
+        auto_register: bool = True,
+    ):
         """Initialize a new Toolkit.
 
         Args:
             name: A descriptive name for the toolkit
+            tools: List of tools to include in the toolkit
+            instructions: Instructions for the toolkit
+            add_instructions: Whether to add instructions to the toolkit
+            include_tools: List of tool names to include in the toolkit
+            exclude_tools: List of tool names to exclude from the toolkit
+            requires_confirmation_tools: List of tool names that require user confirmation
+            external_execution_required_tools: List of tool names that will be executed outside of the agent loop
+            cache_results (bool): Enable in-memory caching of function results.
+            cache_ttl (int): Time-to-live for cached results in seconds.
+            cache_dir (Optional[str]): Directory to store cache files. Defaults to system temp dir.
+            auto_register (bool): Whether to automatically register all methods in the class.
+            stop_after_tool_call_tools (Optional[List[str]]): List of function names that should stop the agent after execution.
+            show_result_tools (Optional[List[str]]): List of function names whose results should be shown.
         """
         self.name: str = name
+        self.tools: List[Callable] = tools
         self.functions: Dict[str, Function] = OrderedDict()
+        self.instructions: Optional[str] = instructions
+        self.add_instructions: bool = add_instructions
+
+        self.requires_confirmation_tools: list[str] = requires_confirmation_tools or []
+        self.external_execution_required_tools: list[str] = external_execution_required_tools or []
+
+        self.stop_after_tool_call_tools: list[str] = stop_after_tool_call_tools or []
+        self.show_result_tools: list[str] = show_result_tools or []
+
+        self._check_tools_filters(
+            available_tools=[tool.__name__ for tool in tools], include_tools=include_tools, exclude_tools=exclude_tools
+        )
+
+        self.include_tools = include_tools
+        self.exclude_tools = exclude_tools
+
+        self.cache_results: bool = cache_results
+        self.cache_ttl: int = cache_ttl
+        self.cache_dir: Optional[str] = cache_dir
+
+        # Automatically register all methods if auto_register is True
+        if auto_register and self.tools:
+            self._register_tools()
+
+    def _check_tools_filters(
+        self,
+        available_tools: List[str],
+        include_tools: Optional[list[str]] = None,
+        exclude_tools: Optional[list[str]] = None,
+    ) -> None:
+        """Check if `include_tools` and `exclude_tools` are valid"""
+        if include_tools or exclude_tools:
+            if include_tools:
+                missing_includes = set(include_tools) - set(available_tools)
+                if missing_includes:
+                    raise ValueError(f"Included tool(s) not present in the toolkit: {', '.join(missing_includes)}")
+
+            if exclude_tools:
+                missing_excludes = set(exclude_tools) - set(available_tools)
+                if missing_excludes:
+                    raise ValueError(f"Excluded tool(s) not present in the toolkit: {', '.join(missing_excludes)}")
+
+        if self.requires_confirmation_tools:
+            missing_requires_confirmation = set(self.requires_confirmation_tools) - set(available_tools)
+            if missing_requires_confirmation:
+                log_warning(
+                    f"Requires confirmation tool(s) not present in the toolkit: {', '.join(missing_requires_confirmation)}"
+                )
 
-    def register(self, function: Callable[..., Any], sanitize_arguments: bool = True):
+        if self.external_execution_required_tools:
+            missing_external_execution_required = set(self.external_execution_required_tools) - set(available_tools)
+            if missing_external_execution_required:
+                log_warning(
+                    f"External execution required tool(s) not present in the toolkit: {', '.join(missing_external_execution_required)}"
+                )
+
+    def _register_tools(self) -> None:
+        """Register all tools."""
+        for tool in self.tools:
+            self.register(tool)
+
+    def register(self, function: Callable[..., Any], name: Optional[str] = None):
         """Register a function with the toolkit.
 
         Args:
             function: The callable to register
+            name: Optional custom name for the function
 
         Returns:
             The registered function
         """
         try:
+            tool_name = name or function.__name__
+            if self.include_tools is not None and tool_name not in self.include_tools:
+                return
+            if self.exclude_tools is not None and tool_name in self.exclude_tools:
+                return
+
             f = Function(
-                name=function.__name__,
+                name=tool_name,
                 entrypoint=function,
-                sanitize_arguments=sanitize_arguments,
+                cache_results=self.cache_results,
+                cache_dir=self.cache_dir,
+                cache_ttl=self.cache_ttl,
+                requires_confirmation=tool_name in self.requires_confirmation_tools,
+                external_execution=tool_name in self.external_execution_required_tools,
+                stop_after_tool_call=tool_name in self.stop_after_tool_call_tools,
+                show_result=tool_name in self.show_result_tools or tool_name in self.stop_after_tool_call_tools,
             )
             self.functions[f.name] = f
-            logger.debug(f"Function: {f.name} registered with {self.name}")
+            log_debug(f"Function: {f.name} registered with {self.name}")
         except Exception as e:
             logger.warning(f"Failed to create Function for: {function.__name__}")
             raise e
 
-    def instructions(self) -> str:
-        return ""
+    @property
+    def requires_connect(self) -> bool:
+        """Whether the toolkit requires connection management."""
+        return self._requires_connect
+
+    def connect(self) -> None:
+        """
+        Establish any required connections for the toolkit.
+        Override this method in subclasses that require connection management.
+        Called automatically by the Agent when _requires_connect is True.
+        """
+        pass
+
+    def close(self) -> None:
+        """
+        Close any open connections for the toolkit.
+        Override this method in subclasses that require connection management.
+        Called automatically by the Agent when _requires_connect is True.
+        """
+        pass
 
     def __repr__(self):
         return f"<{self.__class__.__name__} name={self.name} functions={list(self.functions.keys())}>"

agno/tools/trafilatura.py

@@ -0,0 +1,388 @@
+import json
+from typing import Any, Callable, Dict, List, Optional, Set
+
+from agno.tools import Toolkit
+from agno.utils.log import log_debug, logger
+
+try:
+    from trafilatura import (
+        extract,
+        extract_metadata,
+        fetch_url,
+        html2txt,
+    )
+    from trafilatura.meta import reset_caches
+
+    # Import spider functionality
+    try:
+        from trafilatura.spider import focused_crawler
+
+        SPIDER_AVAILABLE = True
+    except ImportError:
+        SPIDER_AVAILABLE = False
+        logger.warning("Trafilatura spider module not available. Web crawling functionality will be disabled.")
+
+except ImportError:
+    raise ImportError("`trafilatura` not installed. Please install using `pip install trafilatura`")
+
+
+class TrafilaturaTools(Toolkit):
+    """
+    TrafilaturaTools is a toolkit for web scraping and text extraction.
+
+    Args:
+        output_format (str): Default output format for extractions. Options: 'txt', 'json', 'xml', 'markdown', 'csv', 'html', 'xmltei'.
+        include_comments (bool): Whether to extract comments along with main text by default.
+        include_tables (bool): Whether to include table content by default.
+        include_images (bool): Whether to include image information by default (experimental).
+        include_formatting (bool): Whether to preserve formatting by default.
+        include_links (bool): Whether to preserve links by default (experimental).
+        with_metadata (bool): Whether to include metadata in extractions by default.
+        favor_precision (bool): Whether to prefer precision over recall by default.
+        favor_recall (bool): Whether to prefer recall over precision by default.
+        target_language (Optional[str]): Default target language filter (ISO 639-1 format).
+        deduplicate (bool): Whether to remove duplicate segments by default.
+        max_tree_size (Optional[int]): Maximum tree size for processing.
+        max_crawl_urls (int): Maximum number of URLs to crawl per website.
+        max_known_urls (int): Maximum number of known URLs during crawling.
+    """
+
+    def __init__(
+        self,
+        output_format: str = "txt",
+        include_comments: bool = True,
+        include_tables: bool = True,
+        include_images: bool = False,
+        include_formatting: bool = False,
+        include_links: bool = False,
+        with_metadata: bool = False,
+        favor_precision: bool = False,
+        favor_recall: bool = False,
+        target_language: Optional[str] = None,
+        deduplicate: bool = False,
+        max_tree_size: Optional[int] = None,
+        max_crawl_urls: int = 10,
+        max_known_urls: int = 100000,
+        # Tool enable flags for <6 functions
+        enable_extract_text: bool = True,
+        enable_extract_metadata_only: bool = True,
+        enable_html_to_text: bool = True,
+        enable_extract_batch: bool = True,
+        enable_crawl_website: bool = True,
+        all: bool = False,
+        **kwargs,
+    ):
+        self.output_format = output_format
+        self.include_comments = include_comments
+        self.include_tables = include_tables
+        self.include_images = include_images
+        self.include_formatting = include_formatting
+        self.include_links = include_links
+        self.with_metadata = with_metadata
+        self.favor_precision = favor_precision
+        self.favor_recall = favor_recall
+        self.target_language = target_language
+        self.deduplicate = deduplicate
+        self.max_tree_size = max_tree_size
+        self.max_crawl_urls = max_crawl_urls
+        self.max_known_urls = max_known_urls
+
+        tools: List[Callable] = []
+        if all or enable_extract_text:
+            tools.append(self.extract_text)
+        if all or enable_extract_metadata_only:
+            tools.append(self.extract_metadata_only)
+        if all or enable_html_to_text:
+            tools.append(self.html_to_text)
+        if all or enable_extract_batch:
+            tools.append(self.extract_batch)
+
+        if all or enable_crawl_website:
+            if not SPIDER_AVAILABLE:
+                logger.warning("Web crawling requested but spider module not available. Skipping crawler tool.")
+            else:
+                tools.append(self.crawl_website)
+
+        super().__init__(name="trafilatura_tools", tools=tools, **kwargs)
+
+    def _get_extraction_params(
+        self,
+        output_format: Optional[str] = None,
+        include_comments: Optional[bool] = None,
+        include_tables: Optional[bool] = None,
+        include_images: Optional[bool] = None,
+        include_formatting: Optional[bool] = None,
+        include_links: Optional[bool] = None,
+        with_metadata: Optional[bool] = None,
+        favor_precision: Optional[bool] = None,
+        favor_recall: Optional[bool] = None,
+        target_language: Optional[str] = None,
+        deduplicate: Optional[bool] = None,
+        max_tree_size: Optional[int] = None,
+        url_blacklist: Optional[Set[str]] = None,
+        author_blacklist: Optional[Set[str]] = None,
+    ) -> Dict[str, Any]:
+        """Helper method to build extraction parameters with fallbacks to instance defaults."""
+        return {
+            "output_format": output_format if output_format is not None else self.output_format,
+            "include_comments": include_comments if include_comments is not None else self.include_comments,
+            "include_tables": include_tables if include_tables is not None else self.include_tables,
+            "include_images": include_images if include_images is not None else self.include_images,
+            "include_formatting": include_formatting if include_formatting is not None else self.include_formatting,
+            "include_links": include_links if include_links is not None else self.include_links,
+            "with_metadata": with_metadata if with_metadata is not None else self.with_metadata,
+            "favor_precision": favor_precision if favor_precision is not None else self.favor_precision,
+            "favor_recall": favor_recall if favor_recall is not None else self.favor_recall,
+            "target_language": target_language if target_language is not None else self.target_language,
+            "deduplicate": deduplicate if deduplicate is not None else self.deduplicate,
+            "max_tree_size": max_tree_size if max_tree_size is not None else self.max_tree_size,
+            "url_blacklist": url_blacklist,
+            "author_blacklist": author_blacklist,
+        }
+
+    def extract_text(
+        self,
+        url: str,
+        output_format: Optional[str] = None,
+    ) -> str:
+        """
+        Extract main text content from a web page URL using Trafilatura.
+
+        Args:
+            url (str): The URL to extract content from.
+            output_format (Optional[str]): Output format. Options: 'txt', 'json', 'xml', 'markdown', 'csv', 'html', 'xmltei'.
+
+        Returns:
+            str: Extracted content in the specified format, or error message if extraction fails.
+        """
+        try:
+            log_debug(f"Extracting text from URL: {url}")
+
+            # Fetch the webpage content
+            html_content = fetch_url(url)
+            if not html_content:
+                return f"Error: Could not fetch content from URL: {url}"
+
+            # Get extraction parameters
+            params = self._get_extraction_params(output_format=output_format)
+
+            result = extract(html_content, url=url, **params)
+
+            if result is None:
+                return f"Error: Could not extract readable content from URL: {url}"
+
+            # Reset caches
+            reset_caches()
+
+            return result
+
+        except Exception as e:
+            logger.warning(f"Error extracting text from {url}: {e}")
+            return f"Error extracting text from {url}: {e}"
+
+    def extract_metadata_only(
+        self,
+        url: str,
+        as_json: bool = True,
+    ) -> str:
+        """
+        Extract only metadata from a web page URL.
+
+        Args:
+            url (str): The URL to extract metadata from.
+            as_json (bool): Whether to return metadata as JSON string.
+
+        Returns:
+            str: Extracted metadata as JSON string or formatted text.
+        """
+        try:
+            log_debug(f"Extracting metadata from URL: {url}")
+
+            # Fetch the webpage content
+            html_content = fetch_url(url)
+            if not html_content:
+                return f"Error: Could not fetch content from URL: {url}"
+
+            # Extract metadata
+            metadata_doc = extract_metadata(
+                html_content,
+                default_url=url,
+                extensive=True,  # default
+                author_blacklist=None,
+            )
+
+            if metadata_doc is None:
+                return f"Error: Could not extract metadata from URL: {url}"
+
+            metadata_dict = metadata_doc.as_dict()
+
+            # Reset caches
+            reset_caches()
+
+            if as_json:
+                return json.dumps(metadata_dict, indent=2, default=str)
+            else:
+                return "\n".join(f"{key}: {value}" for key, value in metadata_dict.items())
+
+        except Exception as e:
+            logger.warning(f"Error extracting metadata from {url}: {e}")
+            return f"Error extracting metadata from {url}: {e}"
+
+    def crawl_website(
+        self,
+        homepage_url: str,
+        extract_content: bool = False,
+    ) -> str:
+        """
+        Crawl a website and optionally extract content from discovered pages.
+
+        Args:
+            homepage_url (str): The starting URL (preferably homepage) to crawl from.
+            extract_content (bool): Whether to extract content from discovered URLs.
+
+        Returns:
+            str: JSON containing crawl results and optionally extracted content.
+        """
+        if not SPIDER_AVAILABLE:
+            return "Error: Web crawling functionality not available. Trafilatura spider module could not be imported."
+
+        try:
+            log_debug(f"Starting website crawl from: {homepage_url}")
+
+            # Use instance configuration
+            max_seen = self.max_crawl_urls
+            max_known = self.max_known_urls
+            lang = self.target_language
+
+            # Perform focused crawling
+            to_visit, known_links = focused_crawler(
+                homepage=homepage_url,
+                max_seen_urls=max_seen,
+                max_known_urls=max_known,
+                lang=lang,
+            )
+
+            crawl_results = {
+                "homepage": homepage_url,
+                "to_visit": list(to_visit) if to_visit else [],
+                "known_links": list(known_links) if known_links else [],
+                "stats": {
+                    "urls_to_visit": len(to_visit) if to_visit else 0,
+                    "known_links_count": len(known_links) if known_links else 0,
+                },
+            }
+
+            # Optionally extract content from discovered URLs
+            if extract_content and known_links:
+                log_debug("Extracting content from discovered URLs")
+                extracted_content = {}
+
+                # Limit extraction to avoid overwhelming responses
+                urls_to_extract = list(known_links)[: min(10, len(known_links))]
+
+                for url in urls_to_extract:
+                    try:
+                        params = self._get_extraction_params()
+
+                        html_content = fetch_url(url)
+                        if html_content:
+                            content = extract(html_content, url=url, **params)
+                            if content:
+                                extracted_content[url] = content
+                    except Exception as e:
+                        extracted_content[url] = f"Error extracting content: {e}"
+
+                crawl_results["extracted_content"] = extracted_content
+
+            # Reset caches
+            reset_caches()
+
+            return json.dumps(crawl_results, indent=2, default=str)
+
+        except Exception as e:
+            logger.warning(f"Error crawling website {homepage_url}: {e}")
+            return f"Error crawling website {homepage_url}: {e}"
+
+    def html_to_text(
+        self,
+        html_content: str,
+        clean: bool = True,
+    ) -> str:
+        """
+        Convert HTML content to plain text using Trafilatura's html2txt function.
+
+        Args:
+            html_content (str): The HTML content to convert.
+            clean (bool): Whether to remove potentially undesirable elements.
+
+        Returns:
+            str: Plain text extracted from HTML.
+        """
+        try:
+            log_debug("Converting HTML to text")
+
+            result = html2txt(html_content, clean=clean)
+
+            # Reset caches
+            reset_caches()
+
+            return result if result else "Error: Could not extract text from HTML content"
+
+        except Exception as e:
+            logger.warning(f"Error converting HTML to text: {e}")
+            return f"Error converting HTML to text: {e}"
+
+    def extract_batch(
+        self,
+        urls: List[str],
+    ) -> str:
+        """
+        Extract content from multiple URLs in batch.
+
+        Args:
+            urls (List[str]): List of URLs to extract content from.
+
+        Returns:
+            str: JSON containing batch extraction results.
+        """
+        try:
+            log_debug(f"Starting batch extraction for {len(urls)} URLs")
+
+            results = {}
+            failed_urls = []
+
+            for url in urls:
+                try:
+                    params = self._get_extraction_params()
+
+                    html_content = fetch_url(url)
+                    if html_content:
+                        content = extract(html_content, url=url, **params)
+                        if content:
+                            results[url] = content
+                        else:
+                            failed_urls.append(url)
+                    else:
+                        failed_urls.append(url)
+
+                except Exception as e:
+                    failed_urls.append(url)
+                    results[url] = f"Error: {e}"
+
+            # Reset caches after batch processing
+            reset_caches()
+
+            batch_results = {
+                "successful_extractions": len(results)
+                - len([k for k, v in results.items() if str(v).startswith("Error:")]),
+                "failed_extractions": len(failed_urls),
+                "total_urls": len(urls),
+                "results": results,
+                "failed_urls": failed_urls,
+            }
+
+            return json.dumps(batch_results, indent=2, default=str)
+
+        except Exception as e:
+            logger.warning(f"Error in batch extraction: {e}")
+            return f"Error in batch extraction: {e}"

agno/tools/trello.py

@@ -1,9 +1,9 @@
 import json
 from os import getenv
-from typing import Optional
+from typing import Any, List, Optional
 
 from agno.tools import Toolkit
-from agno.utils.log import logger
+from agno.utils.log import log_debug, log_info, logger
 
 try:
     from trello import TrelloClient  # type: ignore
@@ -17,16 +17,8 @@ class TrelloTools(Toolkit):
         api_key: Optional[str] = None,
         api_secret: Optional[str] = None,
         token: Optional[str] = None,
-        create_card: bool = True,
-        get_board_lists: bool = True,
-        move_card: bool = True,
-        get_cards: bool = True,
-        create_board: bool = True,
-        create_list: bool = True,
-        list_boards: bool = True,
+        **kwargs,
     ):
-        super().__init__(name="trello")
-
         self.api_key = api_key or getenv("TRELLO_API_KEY")
         self.api_secret = api_secret or getenv("TRELLO_API_SECRET")
         self.token = token or getenv("TRELLO_TOKEN")
@@ -40,20 +32,17 @@ class TrelloTools(Toolkit):
             logger.error(f"Error initializing Trello client: {e}")
             self.client = None
 
-        if create_card:
-            self.register(self.create_card)
-        if get_board_lists:
-            self.register(self.get_board_lists)
-        if move_card:
-            self.register(self.move_card)
-        if get_cards:
-            self.register(self.get_cards)
-        if create_board:
-            self.register(self.create_board)
-        if create_list:
-            self.register(self.create_list)
-        if list_boards:
-            self.register(self.list_boards)
+        tools: List[Any] = [
+            self.create_card,
+            self.get_board_lists,
+            self.move_card,
+            self.get_cards,
+            self.create_board,
+            self.create_list,
+            self.list_boards,
+        ]
+
+        super().__init__(name="trello", tools=tools, **kwargs)
 
     def create_card(self, board_id: str, list_name: str, card_title: str, description: str = "") -> str:
         """
@@ -72,7 +61,7 @@ class TrelloTools(Toolkit):
             if not self.client:
                 return "Trello client not initialized"
 
-            logger.info(f"Creating card {card_title}")
+            log_info(f"Creating card {card_title}")
 
             board = self.client.get_board(board_id)
             target_list = None
@@ -106,6 +95,8 @@ class TrelloTools(Toolkit):
             if not self.client:
                 return "Trello client not initialized"
 
+            log_debug(f"Getting lists for board {board_id}")
+
             board = self.client.get_board(board_id)
             lists = board.list_lists()
 
@@ -131,6 +122,8 @@ class TrelloTools(Toolkit):
             if not self.client:
                 return "Trello client not initialized"
 
+            log_debug(f"Moving card {card_id} to list {list_id}")
+
             card = self.client.get_card(card_id)
             card.change_list(list_id)
 
@@ -153,6 +146,8 @@ class TrelloTools(Toolkit):
             if not self.client:
                 return "Trello client not initialized"
 
+            log_debug(f"Getting cards for list {list_id}")
+
             trello_list = self.client.get_list(list_id)
             cards = trello_list.list_cards()
 
@@ -187,7 +182,7 @@ class TrelloTools(Toolkit):
             if not self.client:
                 return "Trello client not initialized"
 
-            logger.info(f"Creating board {name}")
+            log_info(f"Creating board {name}")
 
             board = self.client.add_board(board_name=name, default_lists=default_lists)
 
@@ -218,7 +213,7 @@ class TrelloTools(Toolkit):
             if not self.client:
                 return "Trello client not initialized"
 
-            logger.info(f"Creating list {list_name}")
+            log_info(f"Creating list {list_name}")
 
             board = self.client.get_board(board_id)
             new_list = board.add_list(name=list_name, pos=pos)
@@ -250,6 +245,8 @@ class TrelloTools(Toolkit):
             if not self.client:
                 return "Trello client not initialized"
 
+            log_debug(f"Listing boards with filter: {board_filter}")
+
             boards = self.client.list_boards(board_filter=board_filter)
 
             boards_list = []