agno 2.3.16__py3-none-any.whl → 2.3.18__py3-none-any.whl

agno/vectordb/chroma/chromadb.py

@@ -1,7 +1,9 @@
 import asyncio
 import json
+from collections import defaultdict
+from concurrent.futures import ThreadPoolExecutor
 from hashlib import md5
-from typing import Any, Dict, List, Mapping, Optional, Union, cast
+from typing import Any, Dict, List, Mapping, Optional, Tuple, Union, cast
 
 try:
     from chromadb import Client as ChromaDbClient
@@ -20,9 +22,61 @@ from agno.knowledge.reranker.base import Reranker
 from agno.utils.log import log_debug, log_error, log_info, log_warning, logger
 from agno.vectordb.base import VectorDb
 from agno.vectordb.distance import Distance
+from agno.vectordb.search import SearchType
+
+
+def reciprocal_rank_fusion(
+    ranked_lists: List[List[Tuple[str, float]]],
+    k: int = 60,
+) -> List[Tuple[str, float]]:
+    """
+    Combine multiple ranked lists using Reciprocal Rank Fusion (RRF).
+
+    RRF is a simple yet effective method for combining multiple rankings.
+    The formula is: RRF(d) = sum(1 / (k + rank_i(d))) for each ranking i
+
+    Args:
+        ranked_lists: List of ranked results, each as [(doc_id, score), ...]
+        k: RRF constant (default 60, as per original paper by Cormack et al.)
+
+    Returns:
+        Fused ranking as [(doc_id, rrf_score), ...] sorted by score descending
+    """
+    rrf_scores: Dict[str, float] = defaultdict(float)
+
+    for ranked_list in ranked_lists:
+        for rank, (doc_id, _) in enumerate(ranked_list, start=1):
+            rrf_scores[doc_id] += 1.0 / (k + rank)
+
+    sorted_results = sorted(rrf_scores.items(), key=lambda x: x[1], reverse=True)
+    return sorted_results
 
 
 class ChromaDb(VectorDb):
+    """
+    ChromaDb class for managing vector operations with ChromaDB.
+
+    Args:
+        collection: The name of the ChromaDB collection.
+        name: Name of the vector database.
+        description: Description of the vector database.
+        id: Unique identifier for this vector database instance.
+        embedder: The embedder to use when embedding the document contents.
+        distance: The distance metric to use when searching for documents.
+        path: The path to store the ChromaDB data (for persistent client).
+        persistent_client: Whether to use a persistent client.
+        search_type: The search type to use when searching for documents.
+            - SearchType.vector: Pure vector similarity search (default)
+            - SearchType.keyword: Keyword-based search using document content
+            - SearchType.hybrid: Combines vector + FTS with Reciprocal Rank Fusion
+        hybrid_rrf_k: RRF (Reciprocal Rank Fusion) constant for hybrid search.
+            Controls ranking smoothness - higher values give more weight to lower-ranked
+            results, lower values make top results more dominant. Default is 60
+            (per original RRF paper by Cormack et al.).
+        reranker: The reranker to use when reranking documents.
+        **kwargs: Additional arguments to pass to the ChromaDB client.
+    """
+
     def __init__(
         self,
         collection: str,
@@ -33,6 +87,8 @@ class ChromaDb(VectorDb):
         distance: Distance = Distance.cosine,
         path: str = "tmp/chromadb",
         persistent_client: bool = False,
+        search_type: SearchType = SearchType.vector,
+        hybrid_rrf_k: int = 60,
         reranker: Optional[Reranker] = None,
         **kwargs,
     ):
@@ -72,6 +128,10 @@ class ChromaDb(VectorDb):
         self.persistent_client: bool = persistent_client
         self.path: str = path
 
+        # Search type configuration
+        self.search_type: SearchType = search_type
+        self.hybrid_rrf_k: int = hybrid_rrf_k
+
         # Reranker instance
         self.reranker: Optional[Reranker] = reranker
 
@@ -272,7 +332,7 @@ class ChromaDb(VectorDb):
                 embed_tasks = [document.async_embed(embedder=self.embedder) for document in documents]
                 await asyncio.gather(*embed_tasks, return_exceptions=True)
             except Exception as e:
-                log_error(f"Error processing document: {e}")
+                logger.error(f"Error processing document: {e}")
 
         for document in documents:
             cleaned_content = document.content.replace("\x00", "\ufffd")
@@ -502,6 +562,41 @@ class ChromaDb(VectorDb):
         if isinstance(filters, list):
             log_warning("Filter Expressions are not yet supported in ChromaDB. No filters will be applied.")
             filters = None
+
+        if not self._collection:
+            self._collection = self.client.get_collection(name=self.collection_name)
+
+        # Route to appropriate search method based on search_type
+        if self.search_type == SearchType.vector:
+            search_results = self._vector_search(query, limit, filters)
+        elif self.search_type == SearchType.keyword:
+            search_results = self._keyword_search(query, limit, filters)
+        elif self.search_type == SearchType.hybrid:
+            search_results = self._hybrid_search(query, limit, filters)
+        else:
+            logger.error(f"Invalid search type '{self.search_type}'.")
+            return []
+
+        if self.reranker and search_results:
+            try:
+                search_results = self.reranker.rerank(query=query, documents=search_results)
+            except Exception as e:
+                log_warning(f"Reranker failed, returning unranked results: {e}")
+
+        log_info(f"Found {len(search_results)} documents")
+        return search_results
+
+    def _vector_search(self, query: str, limit: int = 5, filters: Optional[Dict[str, Any]] = None) -> List[Document]:
+        """Perform pure vector similarity search.
+
+        Args:
+            query (str): Query to search for.
+            limit (int): Number of results to return.
+            filters (Optional[Dict[str, Any]]): Metadata filters to apply.
+
+        Returns:
+            List[Document]: List of search results.
+        """
         query_embedding = self.embedder.get_embedding(query)
         if query_embedding is None:
             logger.error(f"Error getting embedding for Query: {query}")
@@ -516,11 +611,248 @@ class ChromaDb(VectorDb):
         result: QueryResult = self._collection.query(
             query_embeddings=query_embedding,
             n_results=limit,
-            where=where_filter,  # Add where filter
+            where=where_filter,
             include=["metadatas", "documents", "embeddings", "distances", "uris"],
         )
 
-        # Build search results
+        return self._build_search_results(result)
+
+    def _keyword_search(self, query: str, limit: int = 5, filters: Optional[Dict[str, Any]] = None) -> List[Document]:
+        """Perform keyword-based search using document content filtering.
+
+        This uses ChromaDB's where_document filter with $contains operator
+        for basic full-text search functionality.
+
+        Args:
+            query (str): Query to search for (keywords to match in document content).
+            limit (int): Number of results to return.
+            filters (Optional[Dict[str, Any]]): Metadata filters to apply.
+
+        Returns:
+            List[Document]: List of search results.
+        """
+        if not self._collection:
+            self._collection = self.client.get_collection(name=self.collection_name)
+
+        # Convert simple filters to ChromaDB's format if needed
+        where_filter = self._convert_filters(filters) if filters else None
+
+        # Get first significant word for $contains filter
+        query_words = query.split()
+        if not query_words:
+            return []
+
+        # Use where_document to filter by document content
+        where_document: Dict[str, Any] = {"$contains": query_words[0]}
+
+        try:
+            # Get documents matching the keyword filter
+            result = self._collection.get(
+                where=where_filter,
+                where_document=cast(Any, where_document),
+                limit=limit,
+                include=["metadatas", "documents", "embeddings"],
+            )
+
+            return self._build_get_results(cast(Dict[str, Any], result), query)
+        except Exception as e:
+            logger.error(f"Error in keyword search: {e}")
+            return []
+
+    def _hybrid_search(self, query: str, limit: int = 5, filters: Optional[Dict[str, Any]] = None) -> List[Document]:
+        """Perform hybrid search combining vector similarity with full-text search using RRF.
+
+        This method combines:
+        1. Dense vector similarity search (semantic search)
+        2. Full-text search (keyword/lexical search)
+
+        Results are fused using Reciprocal Rank Fusion (RRF) for optimal ranking.
+
+        Args:
+            query (str): Query to search for.
+            limit (int): Number of results to return.
+            filters (Optional[Dict[str, Any]]): Metadata filters to apply.
+
+        Returns:
+            List[Document]: List of search results with RRF-fused ranking.
+        """
+        query_embedding = self.embedder.get_embedding(query)
+        if query_embedding is None:
+            logger.error(f"Error getting embedding for Query: {query}")
+            return []
+
+        if not self._collection:
+            self._collection = self.client.get_collection(name=self.collection_name)
+
+        # Convert simple filters to ChromaDB's format if needed
+        where_filter = self._convert_filters(filters) if filters else None
+
+        # Fetch more candidates than needed for better fusion
+        fetch_k = min(limit * 3, 100)
+
+        def dense_vector_similarity_search() -> List[Tuple[str, float]]:
+            """Dense vector similarity search."""
+            try:
+                results = self._collection.query(  # type: ignore
+                    query_embeddings=query_embedding,
+                    n_results=fetch_k,
+                    where=where_filter,
+                    include=["documents", "metadatas", "distances"],
+                )
+
+                ranked: List[Tuple[str, float]] = []
+                if results.get("ids") and results["ids"][0]:
+                    for i, doc_id in enumerate(results["ids"][0]):
+                        distance = results["distances"][0][i] if results.get("distances") else 0  # type: ignore
+                        # Convert distance to similarity score (lower distance = higher score)
+                        score = 1.0 / (1.0 + distance)
+                        ranked.append((doc_id, score))
+                return ranked
+            except Exception as e:
+                log_error(f"Error in vector search component: {e}")
+                return []
+
+        def fts_search() -> List[Tuple[str, float]]:
+            """Full-text search using ChromaDB's where_document filter."""
+            try:
+                query_words = query.split()
+                if not query_words:
+                    return []
+
+                # Use first word for $contains filter
+                fts_where_document: Dict[str, Any] = {"$contains": query_words[0]}
+
+                results = self._collection.query(  # type: ignore
+                    query_embeddings=query_embedding,
+                    n_results=fetch_k,
+                    where=where_filter,
+                    where_document=cast(Any, fts_where_document),
+                    include=["documents", "metadatas", "distances"],
+                )
+
+                ranked: List[Tuple[str, float]] = []
+                if results.get("ids") and results["ids"][0]:
+                    for i, doc_id in enumerate(results["ids"][0]):
+                        # Score based on term overlap (simple BM25-like scoring)
+                        doc = results["documents"][0][i] if results.get("documents") else ""  # type: ignore
+                        query_terms = set(query.lower().split())
+                        doc_terms = set(doc.lower().split()) if doc else set()
+                        overlap = len(query_terms & doc_terms)
+                        score = overlap / max(len(query_terms), 1)
+                        ranked.append((doc_id, score))
+
+                # Sort by score descending
+                ranked.sort(key=lambda x: x[1], reverse=True)
+                return ranked
+            except Exception as e:
+                log_error(f"Error in FTS search component: {e}")
+                return []
+
+        # Execute searches in parallel for better performance
+        with ThreadPoolExecutor(max_workers=2) as executor:
+            vector_future = executor.submit(dense_vector_similarity_search)
+            fts_future = executor.submit(fts_search)
+
+            vector_results = vector_future.result()
+            fts_results = fts_future.result()
+
+        # Apply RRF fusion
+        fused_ranking = reciprocal_rank_fusion(
+            [vector_results, fts_results],
+            k=self.hybrid_rrf_k,
+        )
+
+        # Get top IDs from fused ranking
+        top_ids = [doc_id for doc_id, _ in fused_ranking[:limit]]
+
+        if not top_ids:
+            return []
+
+        # Fetch full document data for top results
+        try:
+            full_results = self._collection.get(
+                ids=top_ids,
+                include=["documents", "metadatas", "embeddings"],
+            )
+        except Exception as e:
+            log_error(f"Error fetching full results: {e}")
+            return []
+
+        # Build lookup dict for results
+        doc_lookup: Dict[str, Dict[str, Any]] = {}
+        result_ids = full_results.get("ids", [])
+        result_docs = full_results.get("documents")
+        result_metas = full_results.get("metadatas")
+        result_embeds = full_results.get("embeddings")
+
+        for i, doc_id in enumerate(result_ids if result_ids is not None else []):
+            doc_lookup[doc_id] = {
+                "document": result_docs[i] if result_docs is not None and i < len(result_docs) else None,
+                "metadata": result_metas[i] if result_metas is not None and i < len(result_metas) else None,
+                "embedding": result_embeds[i] if result_embeds is not None and i < len(result_embeds) else None,
+            }
+
+        # Build final results in fused ranking order
+        search_results: List[Document] = []
+        rrf_scores = dict(fused_ranking)
+
+        for doc_id in top_ids:
+            if doc_id not in doc_lookup:
+                continue
+
+            doc_data = doc_lookup[doc_id]
+            doc_metadata = dict(doc_data["metadata"]) if doc_data["metadata"] else {}
+
+            # Add RRF score to metadata
+            doc_metadata["rrf_score"] = rrf_scores.get(doc_id, 0.0)
+
+            # Extract the fields we added to metadata
+            name_val = doc_metadata.pop("name", None)
+            content_id_val = doc_metadata.pop("content_id", None)
+
+            # Convert types to match Document constructor expectations
+            name = str(name_val) if name_val is not None and not isinstance(name_val, str) else name_val
+            content_id = (
+                str(content_id_val)
+                if content_id_val is not None and not isinstance(content_id_val, str)
+                else content_id_val
+            )
+            content = str(doc_data["document"]) if doc_data["document"] is not None else ""
+
+            # Process embedding
+            embedding = None
+            if doc_data["embedding"] is not None:
+                embed_data = doc_data["embedding"]
+                if hasattr(embed_data, "tolist") and callable(getattr(embed_data, "tolist", None)):
+                    try:
+                        embedding = list(cast(Any, embed_data).tolist())
+                    except (AttributeError, TypeError):
+                        embedding = list(embed_data) if isinstance(embed_data, (list, tuple)) else None
+                elif isinstance(embed_data, (list, tuple)):
+                    embedding = [float(x) for x in embed_data if isinstance(x, (int, float))]
+
+            search_results.append(
+                Document(
+                    id=doc_id,
+                    name=name,
+                    meta_data=doc_metadata,
+                    content=content,
+                    embedding=embedding,
+                    content_id=content_id,
+                )
+            )
+
+        return search_results
+
+    def _build_search_results(self, result: QueryResult) -> List[Document]:
+        """Build Document list from ChromaDB QueryResult.
+
+        Args:
+            result: The QueryResult from ChromaDB query.
+
+        Returns:
+            List[Document]: List of Document objects.
+        """
         search_results: List[Document] = []
 
         ids_list = result.get("ids", [[]])  # type: ignore
@@ -529,13 +861,33 @@ class ChromaDb(VectorDb):
         embeddings_list = result.get("embeddings")  # type: ignore
         distances_list = result.get("distances", [[]])  # type: ignore
 
-        if not ids_list or not metadata_list or not documents_list or embeddings_list is None or not distances_list:
+        # Check if we have valid results - handle numpy arrays carefully
+        if ids_list is None or len(ids_list) == 0:
+            return search_results
+        if metadata_list is None or len(metadata_list) == 0:
+            return search_results
+        if documents_list is None or len(documents_list) == 0:
+            return search_results
+        if distances_list is None or len(distances_list) == 0:
             return search_results
 
         ids = ids_list[0]
         metadata = [dict(m) if m else {} for m in metadata_list[0]]  # Convert to mutable dicts
         documents = documents_list[0]
-        embeddings_raw = embeddings_list[0] if embeddings_list else []
+
+        # Handle embeddings - may be None or numpy array
+        embeddings_raw: Any = []
+        if embeddings_list is not None:
+            try:
+                if len(embeddings_list) > 0:
+                    embeddings_raw = embeddings_list[0]
+            except (TypeError, ValueError):
+                # numpy array truth value issue - try direct access
+                try:
+                    embeddings_raw = embeddings_list[0]
+                except Exception:
+                    embeddings_raw = []
+
         embeddings = []
         for e in embeddings_raw:
             if hasattr(e, "tolist") and callable(getattr(e, "tolist", None)):
@@ -549,7 +901,8 @@ class ChromaDb(VectorDb):
                 embeddings.append([float(e)])
             else:
                 embeddings.append([])
-        distances = distances_list[0]
+
+        distances = distances_list[0] if len(distances_list) > 0 else []
 
         for idx, distance in enumerate(distances):
             if idx < len(metadata):
@@ -582,12 +935,95 @@ class ChromaDb(VectorDb):
                     )
                 )
         except Exception as e:
-            logger.error(f"Error building search results: {e}")
+            log_error(f"Error building search results: {e}")
 
-        if self.reranker:
-            search_results = self.reranker.rerank(query=query, documents=search_results)
+        return search_results
+
+    def _build_get_results(self, result: Dict[str, Any], query: str = "") -> List[Document]:
+        """Build Document list from ChromaDB GetResult.
+
+        Args:
+            result: The GetResult from ChromaDB get.
+            query: The original query for scoring.
+
+        Returns:
+            List[Document]: List of Document objects.
+        """
+        search_results: List[Document] = []
+
+        ids = result.get("ids", [])
+        metadatas = result.get("metadatas", [])
+        documents = result.get("documents", [])
+        embeddings_raw = result.get("embeddings")
+
+        # Check ids safely (may be numpy array)
+        if ids is None:
+            return search_results
+        try:
+            if len(ids) == 0:
+                return search_results
+        except (TypeError, ValueError):
+            return search_results
+
+        embeddings = []
+        # Handle embeddings - may be None or numpy array
+        if embeddings_raw is not None:
+            try:
+                for e in embeddings_raw:
+                    if hasattr(e, "tolist") and callable(getattr(e, "tolist", None)):
+                        try:
+                            embeddings.append(list(cast(Any, e).tolist()))
+                        except (AttributeError, TypeError):
+                            embeddings.append(list(e) if isinstance(e, (list, tuple)) else [])
+                    elif isinstance(e, (list, tuple)):
+                        embeddings.append([float(x) for x in e if isinstance(x, (int, float))])
+                    elif isinstance(e, (int, float)):
+                        embeddings.append([float(e)])
+                    else:
+                        embeddings.append([])
+            except (TypeError, ValueError):
+                # numpy array iteration issue
+                embeddings = []
+
+        try:
+            for idx, id_ in enumerate(ids):
+                doc_metadata = dict(metadatas[idx]) if metadatas and idx < len(metadatas) and metadatas[idx] else {}
+                document = documents[idx] if documents and idx < len(documents) else ""
+
+                # Calculate simple keyword score if query provided
+                if query and document:
+                    query_terms = set(query.lower().split())
+                    doc_terms = set(document.lower().split())
+                    overlap = len(query_terms & doc_terms)
+                    doc_metadata["keyword_score"] = overlap / max(len(query_terms), 1)
+
+                # Extract the fields we added to metadata
+                name_val = doc_metadata.pop("name", None)
+                content_id_val = doc_metadata.pop("content_id", None)
+
+                # Convert types to match Document constructor expectations
+                name = str(name_val) if name_val is not None and not isinstance(name_val, str) else name_val
+                content_id = (
+                    str(content_id_val)
+                    if content_id_val is not None and not isinstance(content_id_val, str)
+                    else content_id_val
+                )
+                content = str(document) if document is not None else ""
+                embedding = embeddings[idx] if idx < len(embeddings) else None
+
+                search_results.append(
+                    Document(
+                        id=id_,
+                        name=name,
+                        meta_data=doc_metadata,
+                        content=content,
+                        embedding=embedding,
+                        content_id=content_id,
+                    )
+                )
+        except Exception as e:
+            log_error(f"Error building get results: {e}")
 
-        log_info(f"Found {len(search_results)} documents")
         return search_results
 
     def _convert_filters(self, filters: Dict[str, Any]) -> Dict[str, Any]:
@@ -892,7 +1328,7 @@ class ChromaDb(VectorDb):
                     current_metadatas = []
 
                 if not ids:
-                    logger.debug(f"No documents found with content_id: {content_id}")
+                    log_debug(f"No documents found with content_id: {content_id}")
                     return
 
                 # Flatten the new metadata first
@@ -914,11 +1350,11 @@ class ChromaDb(VectorDb):
                 chroma_metadatas = cast(List[Mapping[str, Union[str, int, float, bool]]], updated_metadatas)
                 chroma_metadatas = [{k: v for k, v in m.items() if k and v} for m in chroma_metadatas]
                 collection.update(ids=ids, metadatas=chroma_metadatas)  # type: ignore
-                logger.debug(f"Updated metadata for {len(ids)} documents with content_id: {content_id}")
+                log_debug(f"Updated metadata for {len(ids)} documents with content_id: {content_id}")
 
             except TypeError as te:
                 if "object of type 'int' has no len()" in str(te):
-                    logger.warning(
+                    log_warning(
                         f"ChromaDB internal error (version 0.5.0 bug): {te}. Cannot update metadata for content_id '{content_id}'."
                     )
                     return
@@ -926,9 +1362,9 @@ class ChromaDb(VectorDb):
                     raise te
 
         except Exception as e:
-            logger.error(f"Error updating metadata for content_id '{content_id}': {e}")
+            log_error(f"Error updating metadata for content_id '{content_id}': {e}")
             raise
 
     def get_supported_search_types(self) -> List[str]:
         """Get the supported search types for this vector database."""
-        return []  # ChromaDb doesn't use SearchType enum
+        return [SearchType.vector, SearchType.keyword, SearchType.hybrid]

agno/vectordb/pgvector/pgvector.py

@@ -643,6 +643,13 @@ class PgVector(VectorDb):
                                 base_id = doc.id or md5(cleaned_content.encode()).hexdigest()
                                 record_id = md5(f"{base_id}_{content_hash}".encode()).hexdigest()
 
+                                if (
+                                    doc.embedding is not None
+                                    and isinstance(doc.embedding, list)
+                                    and len(doc.embedding) == 0
+                                ):
+                                    log_warning(f"Document {idx} '{doc.name}' has empty embedding (length 0)")
+
                                 if (
                                     doc.embedding is not None
                                     and isinstance(doc.embedding, list)

agno/vectordb/redis/redisdb.py

@@ -477,7 +477,7 @@ class RedisDB(VectorDb):
                 vector_field_name="embedding",
                 text=query,
                 text_field_name="content",
-                alpha=self.vector_score_weight,
+                linear_alpha=self.vector_score_weight,
                 return_fields=["id", "name", "content"],
                 num_results=limit,
             )

agno/workflow/__init__.py

@@ -2,6 +2,7 @@ from agno.workflow.agent import WorkflowAgent
 from agno.workflow.condition import Condition
 from agno.workflow.loop import Loop
 from agno.workflow.parallel import Parallel
+from agno.workflow.remote import RemoteWorkflow
 from agno.workflow.router import Router
 from agno.workflow.step import Step
 from agno.workflow.steps import Steps
@@ -11,6 +12,7 @@ from agno.workflow.workflow import Workflow
 __all__ = [
     "Workflow",
     "WorkflowAgent",
+    "RemoteWorkflow",
     "Steps",
     "Step",
     "Loop",

agno/workflow/agent.py

@@ -5,9 +5,9 @@ from typing import TYPE_CHECKING, Any, Callable, Optional
 from agno.agent import Agent
 from agno.models.base import Model
 from agno.run import RunContext
-from agno.workflow.types import WebSocketHandler
 
 if TYPE_CHECKING:
+    from agno.os.managers import WebSocketHandler
     from agno.session.workflow import WorkflowSession
     from agno.workflow.types import WorkflowExecutionInput
 
@@ -190,7 +190,7 @@ Guidelines:
         execution_input: "WorkflowExecutionInput",
         run_context: RunContext,
         stream: bool = False,
-        websocket_handler: Optional[WebSocketHandler] = None,
+        websocket_handler: Optional["WebSocketHandler"] = None,
     ) -> Callable:
         """
         Create the async workflow execution tool that this agent can call.