agno 2.2.13__py3-none-any.whl

agno/tools/trafilatura.py

@@ -0,0 +1,388 @@
+import json
+from typing import Any, Callable, Dict, List, Optional, Set
+
+from agno.tools import Toolkit
+from agno.utils.log import log_debug, logger
+
+try:
+    from trafilatura import (
+        extract,
+        extract_metadata,
+        fetch_url,
+        html2txt,
+    )
+    from trafilatura.meta import reset_caches
+
+    # Import spider functionality
+    try:
+        from trafilatura.spider import focused_crawler
+
+        SPIDER_AVAILABLE = True
+    except ImportError:
+        SPIDER_AVAILABLE = False
+        logger.warning("Trafilatura spider module not available. Web crawling functionality will be disabled.")
+
+except ImportError:
+    raise ImportError("`trafilatura` not installed. Please install using `pip install trafilatura`")
+
+
+class TrafilaturaTools(Toolkit):
+    """
+    TrafilaturaTools is a toolkit for web scraping and text extraction.
+
+    Args:
+        output_format (str): Default output format for extractions. Options: 'txt', 'json', 'xml', 'markdown', 'csv', 'html', 'xmltei'.
+        include_comments (bool): Whether to extract comments along with main text by default.
+        include_tables (bool): Whether to include table content by default.
+        include_images (bool): Whether to include image information by default (experimental).
+        include_formatting (bool): Whether to preserve formatting by default.
+        include_links (bool): Whether to preserve links by default (experimental).
+        with_metadata (bool): Whether to include metadata in extractions by default.
+        favor_precision (bool): Whether to prefer precision over recall by default.
+        favor_recall (bool): Whether to prefer recall over precision by default.
+        target_language (Optional[str]): Default target language filter (ISO 639-1 format).
+        deduplicate (bool): Whether to remove duplicate segments by default.
+        max_tree_size (Optional[int]): Maximum tree size for processing.
+        max_crawl_urls (int): Maximum number of URLs to crawl per website.
+        max_known_urls (int): Maximum number of known URLs during crawling.
+    """
+
+    def __init__(
+        self,
+        output_format: str = "txt",
+        include_comments: bool = True,
+        include_tables: bool = True,
+        include_images: bool = False,
+        include_formatting: bool = False,
+        include_links: bool = False,
+        with_metadata: bool = False,
+        favor_precision: bool = False,
+        favor_recall: bool = False,
+        target_language: Optional[str] = None,
+        deduplicate: bool = False,
+        max_tree_size: Optional[int] = None,
+        max_crawl_urls: int = 10,
+        max_known_urls: int = 100000,
+        # Tool enable flags for <6 functions
+        enable_extract_text: bool = True,
+        enable_extract_metadata_only: bool = True,
+        enable_html_to_text: bool = True,
+        enable_extract_batch: bool = True,
+        enable_crawl_website: bool = True,
+        all: bool = False,
+        **kwargs,
+    ):
+        self.output_format = output_format
+        self.include_comments = include_comments
+        self.include_tables = include_tables
+        self.include_images = include_images
+        self.include_formatting = include_formatting
+        self.include_links = include_links
+        self.with_metadata = with_metadata
+        self.favor_precision = favor_precision
+        self.favor_recall = favor_recall
+        self.target_language = target_language
+        self.deduplicate = deduplicate
+        self.max_tree_size = max_tree_size
+        self.max_crawl_urls = max_crawl_urls
+        self.max_known_urls = max_known_urls
+
+        tools: List[Callable] = []
+        if all or enable_extract_text:
+            tools.append(self.extract_text)
+        if all or enable_extract_metadata_only:
+            tools.append(self.extract_metadata_only)
+        if all or enable_html_to_text:
+            tools.append(self.html_to_text)
+        if all or enable_extract_batch:
+            tools.append(self.extract_batch)
+
+        if all or enable_crawl_website:
+            if not SPIDER_AVAILABLE:
+                logger.warning("Web crawling requested but spider module not available. Skipping crawler tool.")
+            else:
+                tools.append(self.crawl_website)
+
+        super().__init__(name="trafilatura_tools", tools=tools, **kwargs)
+
+    def _get_extraction_params(
+        self,
+        output_format: Optional[str] = None,
+        include_comments: Optional[bool] = None,
+        include_tables: Optional[bool] = None,
+        include_images: Optional[bool] = None,
+        include_formatting: Optional[bool] = None,
+        include_links: Optional[bool] = None,
+        with_metadata: Optional[bool] = None,
+        favor_precision: Optional[bool] = None,
+        favor_recall: Optional[bool] = None,
+        target_language: Optional[str] = None,
+        deduplicate: Optional[bool] = None,
+        max_tree_size: Optional[int] = None,
+        url_blacklist: Optional[Set[str]] = None,
+        author_blacklist: Optional[Set[str]] = None,
+    ) -> Dict[str, Any]:
+        """Helper method to build extraction parameters with fallbacks to instance defaults."""
+        return {
+            "output_format": output_format if output_format is not None else self.output_format,
+            "include_comments": include_comments if include_comments is not None else self.include_comments,
+            "include_tables": include_tables if include_tables is not None else self.include_tables,
+            "include_images": include_images if include_images is not None else self.include_images,
+            "include_formatting": include_formatting if include_formatting is not None else self.include_formatting,
+            "include_links": include_links if include_links is not None else self.include_links,
+            "with_metadata": with_metadata if with_metadata is not None else self.with_metadata,
+            "favor_precision": favor_precision if favor_precision is not None else self.favor_precision,
+            "favor_recall": favor_recall if favor_recall is not None else self.favor_recall,
+            "target_language": target_language if target_language is not None else self.target_language,
+            "deduplicate": deduplicate if deduplicate is not None else self.deduplicate,
+            "max_tree_size": max_tree_size if max_tree_size is not None else self.max_tree_size,
+            "url_blacklist": url_blacklist,
+            "author_blacklist": author_blacklist,
+        }
+
+    def extract_text(
+        self,
+        url: str,
+        output_format: Optional[str] = None,
+    ) -> str:
+        """
+        Extract main text content from a web page URL using Trafilatura.
+
+        Args:
+            url (str): The URL to extract content from.
+            output_format (Optional[str]): Output format. Options: 'txt', 'json', 'xml', 'markdown', 'csv', 'html', 'xmltei'.
+
+        Returns:
+            str: Extracted content in the specified format, or error message if extraction fails.
+        """
+        try:
+            log_debug(f"Extracting text from URL: {url}")
+
+            # Fetch the webpage content
+            html_content = fetch_url(url)
+            if not html_content:
+                return f"Error: Could not fetch content from URL: {url}"
+
+            # Get extraction parameters
+            params = self._get_extraction_params(output_format=output_format)
+
+            result = extract(html_content, url=url, **params)
+
+            if result is None:
+                return f"Error: Could not extract readable content from URL: {url}"
+
+            # Reset caches
+            reset_caches()
+
+            return result
+
+        except Exception as e:
+            logger.warning(f"Error extracting text from {url}: {e}")
+            return f"Error extracting text from {url}: {e}"
+
+    def extract_metadata_only(
+        self,
+        url: str,
+        as_json: bool = True,
+    ) -> str:
+        """
+        Extract only metadata from a web page URL.
+
+        Args:
+            url (str): The URL to extract metadata from.
+            as_json (bool): Whether to return metadata as JSON string.
+
+        Returns:
+            str: Extracted metadata as JSON string or formatted text.
+        """
+        try:
+            log_debug(f"Extracting metadata from URL: {url}")
+
+            # Fetch the webpage content
+            html_content = fetch_url(url)
+            if not html_content:
+                return f"Error: Could not fetch content from URL: {url}"
+
+            # Extract metadata
+            metadata_doc = extract_metadata(
+                html_content,
+                default_url=url,
+                extensive=True,  # default
+                author_blacklist=None,
+            )
+
+            if metadata_doc is None:
+                return f"Error: Could not extract metadata from URL: {url}"
+
+            metadata_dict = metadata_doc.as_dict()
+
+            # Reset caches
+            reset_caches()
+
+            if as_json:
+                return json.dumps(metadata_dict, indent=2, default=str)
+            else:
+                return "\n".join(f"{key}: {value}" for key, value in metadata_dict.items())
+
+        except Exception as e:
+            logger.warning(f"Error extracting metadata from {url}: {e}")
+            return f"Error extracting metadata from {url}: {e}"
+
+    def crawl_website(
+        self,
+        homepage_url: str,
+        extract_content: bool = False,
+    ) -> str:
+        """
+        Crawl a website and optionally extract content from discovered pages.
+
+        Args:
+            homepage_url (str): The starting URL (preferably homepage) to crawl from.
+            extract_content (bool): Whether to extract content from discovered URLs.
+
+        Returns:
+            str: JSON containing crawl results and optionally extracted content.
+        """
+        if not SPIDER_AVAILABLE:
+            return "Error: Web crawling functionality not available. Trafilatura spider module could not be imported."
+
+        try:
+            log_debug(f"Starting website crawl from: {homepage_url}")
+
+            # Use instance configuration
+            max_seen = self.max_crawl_urls
+            max_known = self.max_known_urls
+            lang = self.target_language
+
+            # Perform focused crawling
+            to_visit, known_links = focused_crawler(
+                homepage=homepage_url,
+                max_seen_urls=max_seen,
+                max_known_urls=max_known,
+                lang=lang,
+            )
+
+            crawl_results = {
+                "homepage": homepage_url,
+                "to_visit": list(to_visit) if to_visit else [],
+                "known_links": list(known_links) if known_links else [],
+                "stats": {
+                    "urls_to_visit": len(to_visit) if to_visit else 0,
+                    "known_links_count": len(known_links) if known_links else 0,
+                },
+            }
+
+            # Optionally extract content from discovered URLs
+            if extract_content and known_links:
+                log_debug("Extracting content from discovered URLs")
+                extracted_content = {}
+
+                # Limit extraction to avoid overwhelming responses
+                urls_to_extract = list(known_links)[: min(10, len(known_links))]
+
+                for url in urls_to_extract:
+                    try:
+                        params = self._get_extraction_params()
+
+                        html_content = fetch_url(url)
+                        if html_content:
+                            content = extract(html_content, url=url, **params)
+                            if content:
+                                extracted_content[url] = content
+                    except Exception as e:
+                        extracted_content[url] = f"Error extracting content: {e}"
+
+                crawl_results["extracted_content"] = extracted_content
+
+            # Reset caches
+            reset_caches()
+
+            return json.dumps(crawl_results, indent=2, default=str)
+
+        except Exception as e:
+            logger.warning(f"Error crawling website {homepage_url}: {e}")
+            return f"Error crawling website {homepage_url}: {e}"
+
+    def html_to_text(
+        self,
+        html_content: str,
+        clean: bool = True,
+    ) -> str:
+        """
+        Convert HTML content to plain text using Trafilatura's html2txt function.
+
+        Args:
+            html_content (str): The HTML content to convert.
+            clean (bool): Whether to remove potentially undesirable elements.
+
+        Returns:
+            str: Plain text extracted from HTML.
+        """
+        try:
+            log_debug("Converting HTML to text")
+
+            result = html2txt(html_content, clean=clean)
+
+            # Reset caches
+            reset_caches()
+
+            return result if result else "Error: Could not extract text from HTML content"
+
+        except Exception as e:
+            logger.warning(f"Error converting HTML to text: {e}")
+            return f"Error converting HTML to text: {e}"
+
+    def extract_batch(
+        self,
+        urls: List[str],
+    ) -> str:
+        """
+        Extract content from multiple URLs in batch.
+
+        Args:
+            urls (List[str]): List of URLs to extract content from.
+
+        Returns:
+            str: JSON containing batch extraction results.
+        """
+        try:
+            log_debug(f"Starting batch extraction for {len(urls)} URLs")
+
+            results = {}
+            failed_urls = []
+
+            for url in urls:
+                try:
+                    params = self._get_extraction_params()
+
+                    html_content = fetch_url(url)
+                    if html_content:
+                        content = extract(html_content, url=url, **params)
+                        if content:
+                            results[url] = content
+                        else:
+                            failed_urls.append(url)
+                    else:
+                        failed_urls.append(url)
+
+                except Exception as e:
+                    failed_urls.append(url)
+                    results[url] = f"Error: {e}"
+
+            # Reset caches after batch processing
+            reset_caches()
+
+            batch_results = {
+                "successful_extractions": len(results)
+                - len([k for k, v in results.items() if str(v).startswith("Error:")]),
+                "failed_extractions": len(failed_urls),
+                "total_urls": len(urls),
+                "results": results,
+                "failed_urls": failed_urls,
+            }
+
+            return json.dumps(batch_results, indent=2, default=str)
+
+        except Exception as e:
+            logger.warning(f"Error in batch extraction: {e}")
+            return f"Error in batch extraction: {e}"

agno/tools/trello.py

@@ -0,0 +1,274 @@
+import json
+from os import getenv
+from typing import Any, List, Optional
+
+from agno.tools import Toolkit
+from agno.utils.log import log_debug, log_info, logger
+
+try:
+    from trello import TrelloClient  # type: ignore
+except ImportError:
+    raise ImportError("`py-trello` not installed.")
+
+
+class TrelloTools(Toolkit):
+    def __init__(
+        self,
+        api_key: Optional[str] = None,
+        api_secret: Optional[str] = None,
+        token: Optional[str] = None,
+        **kwargs,
+    ):
+        self.api_key = api_key or getenv("TRELLO_API_KEY")
+        self.api_secret = api_secret or getenv("TRELLO_API_SECRET")
+        self.token = token or getenv("TRELLO_TOKEN")
+
+        if not all([self.api_key, self.api_secret, self.token]):
+            logger.warning("Missing Trello credentials")
+
+        try:
+            self.client = TrelloClient(api_key=self.api_key, api_secret=self.api_secret, token=self.token)
+        except Exception as e:
+            logger.error(f"Error initializing Trello client: {e}")
+            self.client = None
+
+        tools: List[Any] = [
+            self.create_card,
+            self.get_board_lists,
+            self.move_card,
+            self.get_cards,
+            self.create_board,
+            self.create_list,
+            self.list_boards,
+        ]
+
+        super().__init__(name="trello", tools=tools, **kwargs)
+
+    def create_card(self, board_id: str, list_name: str, card_title: str, description: str = "") -> str:
+        """
+        Create a new card in the specified board and list.
+
+        Args:
+            board_id (str): ID of the board to create the card in
+            list_name (str): Name of the list to add the card to
+            card_title (str): Title of the card
+            description (str): Description of the card
+
+        Returns:
+            str: JSON string containing card details or error message
+        """
+        try:
+            if not self.client:
+                return "Trello client not initialized"
+
+            log_info(f"Creating card {card_title}")
+
+            board = self.client.get_board(board_id)
+            target_list = None
+
+            for lst in board.list_lists():
+                if lst.name.lower() == list_name.lower():
+                    target_list = lst
+                    break
+
+            if not target_list:
+                return f"List '{list_name}' not found on board"
+
+            card = target_list.add_card(name=card_title, desc=description)
+
+            return json.dumps({"id": card.id, "name": card.name, "url": card.url, "list": list_name})
+
+        except Exception as e:
+            return f"Error creating card: {e}"
+
+    def get_board_lists(self, board_id: str) -> str:
+        """
+        Get all lists on a board.
+
+        Args:
+            board_id (str): ID of the board
+
+        Returns:
+            str: JSON string containing lists information
+        """
+        try:
+            if not self.client:
+                return "Trello client not initialized"
+
+            log_debug(f"Getting lists for board {board_id}")
+
+            board = self.client.get_board(board_id)
+            lists = board.list_lists()
+
+            lists_info = [{"id": lst.id, "name": lst.name, "cards_count": len(lst.list_cards())} for lst in lists]
+
+            return json.dumps({"lists": lists_info})
+
+        except Exception as e:
+            return f"Error getting board lists: {e}"
+
+    def move_card(self, card_id: str, list_id: str) -> str:
+        """
+        Move a card to a different list.
+
+        Args:
+            card_id (str): ID of the card to move
+            list_id (str): ID of the destination list
+
+        Returns:
+            str: JSON string containing result of the operation
+        """
+        try:
+            if not self.client:
+                return "Trello client not initialized"
+
+            log_debug(f"Moving card {card_id} to list {list_id}")
+
+            card = self.client.get_card(card_id)
+            card.change_list(list_id)
+
+            return json.dumps({"success": True, "card_id": card_id, "new_list_id": list_id})
+
+        except Exception as e:
+            return f"Error moving card: {e}"
+
+    def get_cards(self, list_id: str) -> str:
+        """
+        Get all cards in a list.
+
+        Args:
+            list_id (str): ID of the list
+
+        Returns:
+            str: JSON string containing cards information
+        """
+        try:
+            if not self.client:
+                return "Trello client not initialized"
+
+            log_debug(f"Getting cards for list {list_id}")
+
+            trello_list = self.client.get_list(list_id)
+            cards = trello_list.list_cards()
+
+            cards_info = [
+                {
+                    "id": card.id,
+                    "name": card.name,
+                    "description": card.description,
+                    "url": card.url,
+                    "labels": [label.name for label in card.labels],
+                }
+                for card in cards
+            ]
+
+            return json.dumps({"cards": cards_info})
+
+        except Exception as e:
+            return f"Error getting cards: {e}"
+
+    def create_board(self, name: str, default_lists: bool = False) -> str:
+        """
+        Create a new Trello board.
+
+        Args:
+            name (str): Name of the board
+            default_lists (bool): Whether the default lists should be created
+
+        Returns:
+            str: JSON string containing board details or error message
+        """
+        try:
+            if not self.client:
+                return "Trello client not initialized"
+
+            log_info(f"Creating board {name}")
+
+            board = self.client.add_board(board_name=name, default_lists=default_lists)
+
+            return json.dumps(
+                {
+                    "id": board.id,
+                    "name": board.name,
+                    "url": board.url,
+                }
+            )
+
+        except Exception as e:
+            return f"Error creating board: {e}"
+
+    def create_list(self, board_id: str, list_name: str, pos: str = "bottom") -> str:
+        """
+        Create a new list on a specified board.
+
+        Args:
+            board_id (str): ID of the board to create the list in
+            list_name (str): Name of the new list
+            pos (str): Position of the list - 'top', 'bottom', or a positive number
+
+        Returns:
+            str: JSON string containing list details or error message
+        """
+        try:
+            if not self.client:
+                return "Trello client not initialized"
+
+            log_info(f"Creating list {list_name}")
+
+            board = self.client.get_board(board_id)
+            new_list = board.add_list(name=list_name, pos=pos)
+
+            return json.dumps(
+                {
+                    "id": new_list.id,
+                    "name": new_list.name,
+                    "pos": new_list.pos,
+                    "board_id": board_id,
+                }
+            )
+
+        except Exception as e:
+            return f"Error creating list: {e}"
+
+    def list_boards(self, board_filter: str = "all") -> str:
+        """
+        Get a list of all boards for the authenticated user.
+
+        Args:
+            board_filter (str): Filter for boards. Options: 'all', 'open', 'closed',
+                              'organization', 'public', 'starred'. Defaults to 'all'.
+
+        Returns:
+            str: JSON string containing list of boards
+        """
+        try:
+            if not self.client:
+                return "Trello client not initialized"
+
+            log_debug(f"Listing boards with filter: {board_filter}")
+
+            boards = self.client.list_boards(board_filter=board_filter)
+
+            boards_list = []
+            for board in boards:
+                board_data = {
+                    "id": board.id,
+                    "name": board.name,
+                    "description": getattr(board, "description", ""),
+                    "url": board.url,
+                    "closed": board.closed,
+                    "starred": getattr(board, "starred", False),
+                    "organization": getattr(board, "idOrganization", None),
+                }
+                boards_list.append(board_data)
+
+            return json.dumps(
+                {
+                    "filter_used": board_filter,
+                    "total_boards": len(boards_list),
+                    "boards": boards_list,
+                }
+            )
+
+        except Exception as e:
+            return f"Error listing boards: {e}"