agno 1.7.9__py3-none-any.whl → 1.7.11__py3-none-any.whl

agno/tools/trafilatura.py

@@ -0,0 +1,372 @@
+import json
+from typing import Any, Callable, Dict, List, Optional, Set
+
+from agno.tools import Toolkit
+from agno.utils.log import log_debug, logger
+
+try:
+    from trafilatura import (
+        extract,
+        extract_metadata,
+        fetch_url,
+        html2txt,
+    )
+    from trafilatura.meta import reset_caches
+
+    # Import spider functionality
+    try:
+        from trafilatura.spider import focused_crawler
+
+        SPIDER_AVAILABLE = True
+    except ImportError:
+        SPIDER_AVAILABLE = False
+        logger.warning("Trafilatura spider module not available. Web crawling functionality will be disabled.")
+
+except ImportError:
+    raise ImportError("`trafilatura` not installed. Please install using `pip install trafilatura`")
+
+
+class TrafilaturaTools(Toolkit):
+    """
+    TrafilaturaTools is a toolkit for web scraping and text extraction.
+
+    Args:
+        output_format (str): Default output format for extractions. Options: 'txt', 'json', 'xml', 'markdown', 'csv', 'html', 'xmltei'.
+        include_comments (bool): Whether to extract comments along with main text by default.
+        include_tables (bool): Whether to include table content by default.
+        include_images (bool): Whether to include image information by default (experimental).
+        include_formatting (bool): Whether to preserve formatting by default.
+        include_links (bool): Whether to preserve links by default (experimental).
+        with_metadata (bool): Whether to include metadata in extractions by default.
+        favor_precision (bool): Whether to prefer precision over recall by default.
+        favor_recall (bool): Whether to prefer recall over precision by default.
+        target_language (Optional[str]): Default target language filter (ISO 639-1 format).
+        deduplicate (bool): Whether to remove duplicate segments by default.
+        max_tree_size (Optional[int]): Maximum tree size for processing.
+        max_crawl_urls (int): Maximum number of URLs to crawl per website.
+        max_known_urls (int): Maximum number of known URLs during crawling.
+    """
+
+    def __init__(
+        self,
+        output_format: str = "txt",
+        include_comments: bool = True,
+        include_tables: bool = True,
+        include_images: bool = False,
+        include_formatting: bool = False,
+        include_links: bool = False,
+        with_metadata: bool = False,
+        favor_precision: bool = False,
+        favor_recall: bool = False,
+        target_language: Optional[str] = None,
+        deduplicate: bool = False,
+        max_tree_size: Optional[int] = None,
+        max_crawl_urls: int = 10,
+        max_known_urls: int = 100000,
+        **kwargs,
+    ):
+        self.output_format = output_format
+        self.include_comments = include_comments
+        self.include_tables = include_tables
+        self.include_images = include_images
+        self.include_formatting = include_formatting
+        self.include_links = include_links
+        self.with_metadata = with_metadata
+        self.favor_precision = favor_precision
+        self.favor_recall = favor_recall
+        self.target_language = target_language
+        self.deduplicate = deduplicate
+        self.max_tree_size = max_tree_size
+        self.max_crawl_urls = max_crawl_urls
+        self.max_known_urls = max_known_urls
+
+        tools: List[Callable] = [self.extract_text, self.extract_metadata_only, self.html_to_text, self.extract_batch]
+
+        if not SPIDER_AVAILABLE:
+            logger.warning("Web crawling requested but spider module not available. Skipping crawler tool.")
+        else:
+            tools.append(self.crawl_website)
+
+        super().__init__(name="trafilatura_tools", tools=tools, **kwargs)
+
+    def _get_extraction_params(
+        self,
+        output_format: Optional[str] = None,
+        include_comments: Optional[bool] = None,
+        include_tables: Optional[bool] = None,
+        include_images: Optional[bool] = None,
+        include_formatting: Optional[bool] = None,
+        include_links: Optional[bool] = None,
+        with_metadata: Optional[bool] = None,
+        favor_precision: Optional[bool] = None,
+        favor_recall: Optional[bool] = None,
+        target_language: Optional[str] = None,
+        deduplicate: Optional[bool] = None,
+        max_tree_size: Optional[int] = None,
+        url_blacklist: Optional[Set[str]] = None,
+        author_blacklist: Optional[Set[str]] = None,
+    ) -> Dict[str, Any]:
+        """Helper method to build extraction parameters with fallbacks to instance defaults."""
+        return {
+            "output_format": output_format if output_format is not None else self.output_format,
+            "include_comments": include_comments if include_comments is not None else self.include_comments,
+            "include_tables": include_tables if include_tables is not None else self.include_tables,
+            "include_images": include_images if include_images is not None else self.include_images,
+            "include_formatting": include_formatting if include_formatting is not None else self.include_formatting,
+            "include_links": include_links if include_links is not None else self.include_links,
+            "with_metadata": with_metadata if with_metadata is not None else self.with_metadata,
+            "favor_precision": favor_precision if favor_precision is not None else self.favor_precision,
+            "favor_recall": favor_recall if favor_recall is not None else self.favor_recall,
+            "target_language": target_language if target_language is not None else self.target_language,
+            "deduplicate": deduplicate if deduplicate is not None else self.deduplicate,
+            "max_tree_size": max_tree_size if max_tree_size is not None else self.max_tree_size,
+            "url_blacklist": url_blacklist,
+            "author_blacklist": author_blacklist,
+        }
+
+    def extract_text(
+        self,
+        url: str,
+        output_format: Optional[str] = None,
+    ) -> str:
+        """
+        Extract main text content from a web page URL using Trafilatura.
+
+        Args:
+            url (str): The URL to extract content from.
+            output_format (Optional[str]): Output format. Options: 'txt', 'json', 'xml', 'markdown', 'csv', 'html', 'xmltei'.
+
+        Returns:
+            str: Extracted content in the specified format, or error message if extraction fails.
+        """
+        try:
+            log_debug(f"Extracting text from URL: {url}")
+
+            # Fetch the webpage content
+            html_content = fetch_url(url)
+            if not html_content:
+                return f"Error: Could not fetch content from URL: {url}"
+
+            # Get extraction parameters
+            params = self._get_extraction_params(output_format=output_format)
+
+            result = extract(html_content, url=url, **params)
+
+            if result is None:
+                return f"Error: Could not extract readable content from URL: {url}"
+
+            # Reset caches
+            reset_caches()
+
+            return result
+
+        except Exception as e:
+            logger.warning(f"Error extracting text from {url}: {e}")
+            return f"Error extracting text from {url}: {e}"
+
+    def extract_metadata_only(
+        self,
+        url: str,
+        as_json: bool = True,
+    ) -> str:
+        """
+        Extract only metadata from a web page URL.
+
+        Args:
+            url (str): The URL to extract metadata from.
+            as_json (bool): Whether to return metadata as JSON string.
+
+        Returns:
+            str: Extracted metadata as JSON string or formatted text.
+        """
+        try:
+            log_debug(f"Extracting metadata from URL: {url}")
+
+            # Fetch the webpage content
+            html_content = fetch_url(url)
+            if not html_content:
+                return f"Error: Could not fetch content from URL: {url}"
+
+            # Extract metadata
+            metadata_doc = extract_metadata(
+                html_content,
+                default_url=url,
+                extensive=True,  # default
+                author_blacklist=None,
+            )
+
+            if metadata_doc is None:
+                return f"Error: Could not extract metadata from URL: {url}"
+
+            metadata_dict = metadata_doc.as_dict()
+
+            # Reset caches
+            reset_caches()
+
+            if as_json:
+                return json.dumps(metadata_dict, indent=2, default=str)
+            else:
+                return "\n".join(f"{key}: {value}" for key, value in metadata_dict.items())
+
+        except Exception as e:
+            logger.warning(f"Error extracting metadata from {url}: {e}")
+            return f"Error extracting metadata from {url}: {e}"
+
+    def crawl_website(
+        self,
+        homepage_url: str,
+        extract_content: bool = False,
+    ) -> str:
+        """
+        Crawl a website and optionally extract content from discovered pages.
+
+        Args:
+            homepage_url (str): The starting URL (preferably homepage) to crawl from.
+            extract_content (bool): Whether to extract content from discovered URLs.
+
+        Returns:
+            str: JSON containing crawl results and optionally extracted content.
+        """
+        if not SPIDER_AVAILABLE:
+            return "Error: Web crawling functionality not available. Trafilatura spider module could not be imported."
+
+        try:
+            log_debug(f"Starting website crawl from: {homepage_url}")
+
+            # Use instance configuration
+            max_seen = self.max_crawl_urls
+            max_known = self.max_known_urls
+            lang = self.target_language
+
+            # Perform focused crawling
+            to_visit, known_links = focused_crawler(
+                homepage=homepage_url,
+                max_seen_urls=max_seen,
+                max_known_urls=max_known,
+                lang=lang,
+            )
+
+            crawl_results = {
+                "homepage": homepage_url,
+                "to_visit": list(to_visit) if to_visit else [],
+                "known_links": list(known_links) if known_links else [],
+                "stats": {
+                    "urls_to_visit": len(to_visit) if to_visit else 0,
+                    "known_links_count": len(known_links) if known_links else 0,
+                },
+            }
+
+            # Optionally extract content from discovered URLs
+            if extract_content and known_links:
+                log_debug("Extracting content from discovered URLs")
+                extracted_content = {}
+
+                # Limit extraction to avoid overwhelming responses
+                urls_to_extract = list(known_links)[: min(10, len(known_links))]
+
+                for url in urls_to_extract:
+                    try:
+                        params = self._get_extraction_params()
+
+                        html_content = fetch_url(url)
+                        if html_content:
+                            content = extract(html_content, url=url, **params)
+                            if content:
+                                extracted_content[url] = content
+                    except Exception as e:
+                        extracted_content[url] = f"Error extracting content: {e}"
+
+                crawl_results["extracted_content"] = extracted_content
+
+            # Reset caches
+            reset_caches()
+
+            return json.dumps(crawl_results, indent=2, default=str)
+
+        except Exception as e:
+            logger.warning(f"Error crawling website {homepage_url}: {e}")
+            return f"Error crawling website {homepage_url}: {e}"
+
+    def html_to_text(
+        self,
+        html_content: str,
+        clean: bool = True,
+    ) -> str:
+        """
+        Convert HTML content to plain text using Trafilatura's html2txt function.
+
+        Args:
+            html_content (str): The HTML content to convert.
+            clean (bool): Whether to remove potentially undesirable elements.
+
+        Returns:
+            str: Plain text extracted from HTML.
+        """
+        try:
+            log_debug("Converting HTML to text")
+
+            result = html2txt(html_content, clean=clean)
+
+            # Reset caches
+            reset_caches()
+
+            return result if result else "Error: Could not extract text from HTML content"
+
+        except Exception as e:
+            logger.warning(f"Error converting HTML to text: {e}")
+            return f"Error converting HTML to text: {e}"
+
+    def extract_batch(
+        self,
+        urls: List[str],
+    ) -> str:
+        """
+        Extract content from multiple URLs in batch.
+
+        Args:
+            urls (List[str]): List of URLs to extract content from.
+
+        Returns:
+            str: JSON containing batch extraction results.
+        """
+        try:
+            log_debug(f"Starting batch extraction for {len(urls)} URLs")
+
+            results = {}
+            failed_urls = []
+
+            for url in urls:
+                try:
+                    params = self._get_extraction_params()
+
+                    html_content = fetch_url(url)
+                    if html_content:
+                        content = extract(html_content, url=url, **params)
+                        if content:
+                            results[url] = content
+                        else:
+                            failed_urls.append(url)
+                    else:
+                        failed_urls.append(url)
+
+                except Exception as e:
+                    failed_urls.append(url)
+                    results[url] = f"Error: {e}"
+
+            # Reset caches after batch processing
+            reset_caches()
+
+            batch_results = {
+                "successful_extractions": len(results)
+                - len([k for k, v in results.items() if str(v).startswith("Error:")]),
+                "failed_extractions": len(failed_urls),
+                "total_urls": len(urls),
+                "results": results,
+                "failed_urls": failed_urls,
+            }
+
+            return json.dumps(batch_results, indent=2, default=str)
+
+        except Exception as e:
+            logger.warning(f"Error in batch extraction: {e}")
+            return f"Error in batch extraction: {e}"

agno/vectordb/clickhouse/clickhousedb.py

@@ -140,7 +140,7 @@ class Clickhouse(VectorDb):
 
             if isinstance(self.index, HNSW):
                 index = (
-                    f"INDEX embedding_index embedding TYPE vector_similarity('hnsw', 'L2Distance', {self.index.quantization}, "
+                    f"INDEX embedding_index embedding TYPE vector_similarity('hnsw', 'L2Distance', {self.embedder.dimensions}, {self.index.quantization}, "
                     f"{self.index.hnsw_max_connections_per_layer}, {self.index.hnsw_candidate_list_size_for_construction})"
                 )
                 self.client.command("SET allow_experimental_vector_similarity_index = 1")

agno/vectordb/milvus/milvus.py

@@ -568,7 +568,7 @@ class Milvus(VectorDb):
         self, query: str, limit: int = 5, filters: Optional[Dict[str, Any]] = None
     ) -> List[Document]:
         if self.search_type == SearchType.hybrid:
-            return self.hybrid_search(query, limit, filters)
+            return await self.async_hybrid_search(query, limit, filters)
 
         query_embedding = self.embedder.get_embedding(query)
         if query_embedding is None:
@@ -691,6 +691,94 @@ class Milvus(VectorDb):
             logger.error(f"Error during hybrid search: {e}")
             return []
 
+    async def async_hybrid_search(
+        self, query: str, limit: int = 5, filters: Optional[Dict[str, Any]] = None
+    ) -> List[Document]:
+        """
+        Perform an asynchronous hybrid search combining dense and sparse vector similarity.
+
+        Args:
+            query (str): Query string to search for
+            limit (int): Maximum number of results to return
+            filters (Optional[Dict[str, Any]]): Filters to apply to the search
+
+        Returns:
+            List[Document]: List of matching documents
+        """
+        from pymilvus import AnnSearchRequest, RRFRanker
+
+        # Get query embeddings
+        dense_vector = self.embedder.get_embedding(query)
+        sparse_vector = self._get_sparse_vector(query)
+
+        if dense_vector is None:
+            logger.error(f"Error getting dense embedding for Query: {query}")
+            return []
+
+        try:
+            # Refer to docs for details- https://milvus.io/docs/multi-vector-search.md
+
+            # Create search request for dense vectors
+            dense_search_param = {
+                "data": [dense_vector],
+                "anns_field": "dense_vector",
+                "param": {"metric_type": self._get_metric_type(), "params": {"nprobe": 10}},
+                "limit": limit
+                * 2,  # Fetch more candidates for better reranking quality - each vector search returns 2x results which are then merged and reranked
+            }
+
+            # Create search request for sparse vectors
+            sparse_search_param = {
+                "data": [sparse_vector],
+                "anns_field": "sparse_vector",
+                "param": {"metric_type": "IP", "params": {"drop_ratio_build": 0.2}},
+                "limit": limit * 2,  # Match dense search limit to ensure balanced candidate pool for reranking
+            }
+
+            # Create search requests
+            dense_request = AnnSearchRequest(**dense_search_param)
+            sparse_request = AnnSearchRequest(**sparse_search_param)
+            reqs = [dense_request, sparse_request]
+
+            # Use RRFRanker for balanced importance between vectors
+            ranker = RRFRanker(60)  # Default k=60
+
+            log_info("Performing async hybrid search")
+            results = await self.async_client.hybrid_search(
+                collection_name=self.collection, reqs=reqs, ranker=ranker, limit=limit, output_fields=["*"]
+            )
+
+            # Build search results
+            search_results: List[Document] = []
+            for hits in results:
+                for hit in hits:
+                    entity = hit.get("entity", {})
+                    meta_data = json.loads(entity.get("meta_data", "{}")) if entity.get("meta_data") else {}
+                    usage = json.loads(entity.get("usage", "{}")) if entity.get("usage") else None
+
+                    search_results.append(
+                        Document(
+                            id=hit.get("id"),
+                            name=entity.get("name", None),
+                            meta_data=meta_data,  # Now a dictionary
+                            content=entity.get("content", ""),
+                            embedder=self.embedder,
+                            embedding=entity.get("dense_vector", None),
+                            usage=usage,  # Now a dictionary or None
+                        )
+                    )
+
+            # Apply additional reranking if custom reranker is provided
+            if self.reranker and search_results:
+                search_results = self.reranker.rerank(query=query, documents=search_results)
+
+            log_info(f"Found {len(search_results)} documents")
+            return search_results
+
+        except Exception as e:
+            logger.error(f"Error during async hybrid search: {e}")
+            return []
+
     def drop(self) -> None:
         if self.exists():
             log_debug(f"Deleting collection: {self.collection}")

agno/vectordb/weaviate/weaviate.py

@@ -2,7 +2,7 @@ import json
 import uuid
 from hashlib import md5
 from os import getenv
-from typing import Any, Dict, List, Optional
+from typing import Any, Dict, List, Optional, Tuple
 
 try:
     from warnings import filterwarnings
@@ -73,6 +73,13 @@ class Weaviate(VectorDb):
         self.reranker: Optional[Reranker] = reranker
         self.hybrid_search_alpha = hybrid_search_alpha
 
+    @staticmethod
+    def _get_doc_uuid(document: Document) -> Tuple[uuid.UUID, str]:
+        cleaned_content = document.content.replace("\x00", "\ufffd")
+        content_hash = md5(cleaned_content.encode()).hexdigest()
+        doc_uuid = uuid.UUID(hex=content_hash[:32])
+        return doc_uuid, cleaned_content
+
     def get_client(self) -> weaviate.WeaviateClient:
         """Initialize and return a Weaviate client instance.
 
@@ -118,7 +125,7 @@ class Weaviate(VectorDb):
             await self.async_client.connect()  # type: ignore
 
         if not await self.async_client.is_ready():  # type: ignore
-            raise Exception("Weaviate async client is not ready")
+            raise ConnectionError("Weaviate async client is not ready")
 
         return self.async_client  # type: ignore
 
@@ -155,6 +162,54 @@ class Weaviate(VectorDb):
         finally:
             await client.close()
 
+    def doc_content_changed(self, document: Document, check_existing: Optional[bool] = True) -> Optional[bool]:
+        """
+        Check if the content of the document has changed by comparing its UUID.
+
+        Args:
+            document (Document): Document to check
+
+        Returns:
+            bool: True if the document content has changed, False otherwise. None on wrong input.
+            check_existing (bool): If True, check if the document exists before checking if the content changed.
+        """
+        if not document or not document.content:
+            logger.warning("Invalid document: Missing content.")
+            return None
+
+        if check_existing and document.name and not self.name_exists(document.name):
+            logger.warning(f"A document by this name does not exist: {document.name}")
+            return None
+
+        doc_uuid, _ = self._get_doc_uuid(document)
+
+        collection = self.get_client().collections.get(self.collection)
+        existing_doc = collection.query.fetch_object_by_id(doc_uuid)
+
+        if not existing_doc:
+            return True
+        else:
+            return False
+
+    def doc_delete(self, name: str) -> None:
+        """
+        Delete all documents from Weaviate with a specific 'name' property.
+
+        Args:
+            name (str): Document name to delete.
+        """
+        collection = self.get_client().collections.get(self.collection)
+        filter_expr = Filter.by_property("name").equal(name)
+
+        result = collection.data.delete_many(where=filter_expr)
+
+        log_debug(f"Deleted document by name: '{name}' - {result.successful} documents deleted.")
+        if result.failed > 0:
+            logger.warning(
+                f"Failed to delete (some chunks of) document with name: '{name}' - "
+                f"Failed {result.failed} out of {result.matches} times. {result.successful} successful deletions."
+            )
+
     def doc_exists(self, document: Document) -> bool:
         """
         Validate if the document exists using consistent UUID generation.
@@ -169,9 +224,7 @@ class Weaviate(VectorDb):
             logger.warning("Invalid document: Missing content.")
             return False  # Early exit for invalid input
 
-        cleaned_content = document.content.replace("\x00", "\ufffd")
-        content_hash = md5(cleaned_content.encode()).hexdigest()
-        doc_uuid = uuid.UUID(hex=content_hash[:32])
+        doc_uuid, _ = self._get_doc_uuid(document)
 
         collection = self.get_client().collections.get(self.collection)
         return collection.data.exists(doc_uuid)
@@ -190,9 +243,7 @@ class Weaviate(VectorDb):
             logger.warning("Invalid document: Missing content.")
             return False  # Early exit for invalid input
 
-        cleaned_content = document.content.replace("\x00", "\ufffd")
-        content_hash = md5(cleaned_content.encode()).hexdigest()
-        doc_uuid = uuid.UUID(hex=content_hash[:32])
+        doc_uuid, _ = self._get_doc_uuid(document)
 
         client = await self.get_async_client()
         try:
@@ -256,9 +307,7 @@ class Weaviate(VectorDb):
                 logger.error(f"Document embedding is None: {document.name}")
                 continue
 
-            cleaned_content = document.content.replace("\x00", "\ufffd")
-            content_hash = md5(cleaned_content.encode()).hexdigest()
-            doc_uuid = uuid.UUID(hex=content_hash[:32])
+            doc_uuid, cleaned_content = self._get_doc_uuid(document)
 
             # Merge filters with metadata
             meta_data = document.meta_data or {}
@@ -305,9 +354,7 @@ class Weaviate(VectorDb):
                         continue
 
                     # Clean content and generate UUID
-                    cleaned_content = document.content.replace("\x00", "\ufffd")
-                    content_hash = md5(cleaned_content.encode()).hexdigest()
-                    doc_uuid = uuid.UUID(hex=content_hash[:32])
+                    doc_uuid, cleaned_content = self._get_doc_uuid(document)
 
                     # Serialize meta_data to JSON string
                     meta_data_str = json.dumps(document.meta_data) if document.meta_data else None
@@ -338,7 +385,28 @@ class Weaviate(VectorDb):
             filters (Optional[Dict[str, Any]]): Filters to apply while upserting
         """
         log_debug(f"Upserting {len(documents)} documents into Weaviate.")
-        self.insert(documents)
+
+        _docs_to_insert = []
+        for document in documents:
+            assert document.name is not None, "Document name must be set for upsert operation."
+
+            if self.name_exists(document.name):
+                if self.doc_content_changed(document, check_existing=False):
+                    log_debug(
+                        f"Document already exists, but content changed. Document will be deleted and added again: {document.name}"
+                    )
+
+                    is_first_or_only_chunk = ("chunk" in document.meta_data and document.meta_data["chunk"] == 1) or (
+                        "chunk" not in document.meta_data
+                    )
+                    if is_first_or_only_chunk:
+                        self.doc_delete(document.name)
+                    _docs_to_insert.append(document)
+                else:
+                    log_debug(f"Document skipped, content is unchanged: {document.name}")
+            else:
+                _docs_to_insert.append(document)
+        self.insert(_docs_to_insert)
 
     async def async_upsert(self, documents: List[Document], filters: Optional[Dict[str, Any]] = None) -> None:
         """
@@ -365,9 +433,7 @@ class Weaviate(VectorDb):
                     logger.error(f"Document embedding is None: {document.name}")
                     continue
 
-                cleaned_content = document.content.replace("\x00", "\ufffd")
-                content_hash = md5(cleaned_content.encode()).hexdigest()
-                doc_uuid = uuid.UUID(hex=content_hash[:32])
+                doc_uuid, cleaned_content = self._get_doc_uuid(document)
 
                 # Serialize meta_data to JSON string
                 meta_data_str = json.dumps(document.meta_data) if document.meta_data else None

agno/workflow/workflow.py

@@ -369,6 +369,9 @@ class Workflow:
         if self.storage is not None:
             self.storage.mode = "workflow"
 
+    def initialize_workflow(self):
+        self.set_storage_mode()
+
     def set_workflow_id(self) -> str:
         if self.workflow_id is None:
             self.workflow_id = str(uuid4())