agno 2.2.13__py3-none-any.whl → 2.4.3__py3-none-any.whl

agno/vectordb/lancedb/lance_db.py

@@ -260,39 +260,6 @@ class LanceDb(VectorDb):
             tbl = self.connection.create_table(name=self.table_name, schema=schema, mode="overwrite", exist_ok=True)  # type: ignore
         return tbl  # type: ignore
 
-    def doc_exists(self, document: Document) -> bool:
-        """
-        Validating if the document exists or not
-
-        Args:
-            document (Document): Document to validate
-        """
-        try:
-            if self.table is not None:
-                cleaned_content = document.content.replace("\x00", "\ufffd")
-                doc_id = md5(cleaned_content.encode()).hexdigest()
-                result = self.table.search().where(f"{self._id}='{doc_id}'").to_arrow()
-                return len(result) > 0
-        except Exception:
-            # Search sometimes fails with stale cache data, it means the doc doesn't exist
-            return False
-
-        return False
-
-    async def async_doc_exists(self, document: Document) -> bool:
-        """
-        Asynchronously validate if the document exists
-
-        Args:
-            document (Document): Document to validate
-
-        Returns:
-            bool: True if document exists, False otherwise
-        """
-        if self.connection:
-            self.table = self.connection.open_table(name=self.table_name)
-        return self.doc_exists(document)
-
     def insert(self, content_hash: str, documents: List[Document], filters: Optional[Dict[str, Any]] = None) -> None:
         """
         Insert documents into the database.
@@ -309,18 +276,20 @@ class LanceDb(VectorDb):
         data = []
 
         for document in documents:
-            if self.doc_exists(document):
-                continue
-
             # Add filters to document metadata if provided
             if filters:
                 meta_data = document.meta_data.copy() if document.meta_data else {}
                 meta_data.update(filters)
                 document.meta_data = meta_data
 
-            document.embed(embedder=self.embedder)
+            # Only embed if the document doesn't already have an embedding
+            # This prevents duplicate embedding when called from async_insert or async_upsert
+            if document.embedding is None:
+                document.embed(embedder=self.embedder)
             cleaned_content = document.content.replace("\x00", "\ufffd")
-            doc_id = str(md5(cleaned_content.encode()).hexdigest())
+            # Include content_hash in ID to ensure uniqueness across different content hashes
+            base_id = document.id or md5(cleaned_content.encode()).hexdigest()
+            doc_id = str(md5(f"{base_id}_{content_hash}".encode()).hexdigest())
             payload = {
                 "name": document.name,
                 "meta_data": document.meta_data,

agno/vectordb/lightrag/lightrag.py

@@ -109,7 +109,7 @@ class LightRag(VectorDb):
             async with httpx.AsyncClient(timeout=30.0) as client:
                 response = await client.post(
                     f"{self.server_url}/query",
-                    json={"query": query, "mode": "hybrid"},
+                    json={"query": query, "mode": "hybrid", "include_references": True},
                     headers=self._get_headers(),
                 )
 
@@ -322,7 +322,7 @@ class LightRag(VectorDb):
             async with httpx.AsyncClient(timeout=30.0) as client:
                 response = await client.post(
                     f"{self.server_url}/query",
-                    json={"query": query, "mode": "hybrid"},
+                    json={"query": query, "mode": "hybrid", "include_references": True},
                     headers=self._get_headers(),
                 )
 
@@ -349,10 +349,11 @@ class LightRag(VectorDb):
         # LightRAG server returns a dict with 'response' key, but we expect a list of documents
         # Convert the response to the expected format
         if isinstance(result, dict) and "response" in result:
-            # Wrap the response in a Document object
-            return [
-                Document(content=result["response"], meta_data={"source": "lightrag", "query": query, "mode": mode})
-            ]
+            meta_data = {"source": "lightrag", "query": query, "mode": mode}
+            # Preserve references from LightRAG response for document citations
+            if "references" in result:
+                meta_data["references"] = result["references"]
+            return [Document(content=result["response"], meta_data=meta_data)]
         elif isinstance(result, list):
             # Convert list items to Document objects
             documents = []

agno/vectordb/milvus/milvus.py

@@ -89,7 +89,7 @@ class Milvus(VectorDb):
             from agno.knowledge.embedder.openai import OpenAIEmbedder
 
             embedder = OpenAIEmbedder()
-            log_info("Embedder not provided, using OpenAIEmbedder as default.")
+            log_debug("Embedder not provided, using OpenAIEmbedder as default.")
         self.embedder: Embedder = embedder
         self.dimensions: Optional[int] = self.embedder.dimensions
 
@@ -229,7 +229,9 @@ class Milvus(VectorDb):
         """
 
         cleaned_content = document.content.replace("\x00", "\ufffd")
-        doc_id = md5(cleaned_content.encode()).hexdigest()
+        # Include content_hash in ID to ensure uniqueness across different content hashes
+        base_id = document.id or md5(cleaned_content.encode()).hexdigest()
+        doc_id = md5(f"{base_id}_{content_hash}".encode()).hexdigest()
 
         # Convert dictionary fields to JSON strings
         meta_data_str = json.dumps(document.meta_data) if document.meta_data else "{}"
@@ -239,7 +241,7 @@ class Milvus(VectorDb):
             "id": doc_id,
             "text": cleaned_content,
             "name": document.name,
-            "content_id": document.content_id,
+            "content_id": document.content_id or "",
             "meta_data": meta_data_str,
             "content": cleaned_content,
             "usage": usage_str,
@@ -317,36 +319,6 @@ class Milvus(VectorDb):
                     max_length=65_535,
                 )
 
-    def doc_exists(self, document: Document) -> bool:
-        """
-        Validating if the document exists or not
-
-        Args:
-            document (Document): Document to validate
-        """
-        if self.client:
-            cleaned_content = document.content.replace("\x00", "\ufffd")
-            doc_id = md5(cleaned_content.encode()).hexdigest()
-            collection_points = self.client.get(
-                collection_name=self.collection,
-                ids=[doc_id],
-            )
-            return len(collection_points) > 0
-        return False
-
-    async def async_doc_exists(self, document: Document) -> bool:
-        """
-        Check if document exists asynchronously.
-        AsyncMilvusClient supports get().
-        """
-        cleaned_content = document.content.replace("\x00", "\ufffd")
-        doc_id = md5(cleaned_content.encode()).hexdigest()
-        collection_points = await self.async_client.get(
-            collection_name=self.collection,
-            ids=[doc_id],
-        )
-        return len(collection_points) > 0
-
     def name_exists(self, name: str) -> bool:
         """
         Validates if a document with the given name exists in the collection.
@@ -362,6 +334,7 @@ class Milvus(VectorDb):
             scroll_result = self.client.query(
                 collection_name=self.collection,
                 filter=expr,
+                output_fields=["id"],
                 limit=1,
             )
             return len(scroll_result) > 0 and len(scroll_result[0]) > 0
@@ -391,6 +364,7 @@ class Milvus(VectorDb):
             scroll_result = self.client.query(
                 collection_name=self.collection,
                 filter=expr,
+                output_fields=["id"],
                 limit=1,
             )
             return len(scroll_result) > 0 and len(scroll_result[0]) > 0
@@ -457,7 +431,7 @@ class Milvus(VectorDb):
                     "id": doc_id,
                     "vector": document.embedding,
                     "name": document.name,
-                    "content_id": document.content_id,
+                    "content_id": document.content_id or "",
                     "meta_data": meta_data,
                     "content": cleaned_content,
                     "usage": document.usage,
@@ -528,7 +502,9 @@ class Milvus(VectorDb):
                     log_debug(f"Skipping document without embedding: {document.name} ({document.meta_data})")
                     return None
                 cleaned_content = document.content.replace("\x00", "\ufffd")
-                doc_id = md5(cleaned_content.encode()).hexdigest()
+                # Include content_hash in ID to ensure uniqueness across different content hashes
+                base_id = document.id or md5(cleaned_content.encode()).hexdigest()
+                doc_id = md5(f"{base_id}_{content_hash}".encode()).hexdigest()
 
                 meta_data = document.meta_data or {}
                 if filters:
@@ -538,7 +514,7 @@ class Milvus(VectorDb):
                     "id": doc_id,
                     "vector": document.embedding,
                     "name": document.name,
-                    "content_id": document.content_id,
+                    "content_id": document.content_id or "",
                     "meta_data": meta_data,
                     "content": cleaned_content,
                     "usage": document.usage,
@@ -573,30 +549,41 @@ class Milvus(VectorDb):
             filters (Optional[Dict[str, Any]]): Filters to apply while upserting
         """
         log_debug(f"Upserting {len(documents)} documents")
-        for document in documents:
-            document.embed(embedder=self.embedder)
-            cleaned_content = document.content.replace("\x00", "\ufffd")
-            doc_id = md5(cleaned_content.encode()).hexdigest()
-
-            meta_data = document.meta_data or {}
-            if filters:
-                meta_data.update(filters)
-
-            data = {
-                "id": doc_id,
-                "vector": document.embedding,
-                "name": document.name,
-                "content_id": document.content_id,
-                "meta_data": document.meta_data,
-                "content": cleaned_content,
-                "usage": document.usage,
-                "content_hash": content_hash,
-            }
-            self.client.upsert(
-                collection_name=self.collection,
-                data=data,
-            )
-            log_debug(f"Upserted document: {document.name} ({document.meta_data})")
+
+        if self.search_type == SearchType.hybrid:
+            for document in documents:
+                document.embed(embedder=self.embedder)
+                data = self._prepare_document_data(content_hash=content_hash, document=document, include_vectors=True)
+                self.client.upsert(
+                    collection_name=self.collection,
+                    data=data,
+                )
+                log_debug(f"Upserted hybrid document: {document.name} ({document.meta_data})")
+        else:
+            for document in documents:
+                document.embed(embedder=self.embedder)
+                cleaned_content = document.content.replace("\x00", "\ufffd")
+                doc_id = md5(cleaned_content.encode()).hexdigest()
+
+                meta_data = document.meta_data or {}
+                if filters:
+                    meta_data.update(filters)
+
+                data = {
+                    "id": doc_id,
+                    "vector": document.embedding,
+                    "name": document.name,
+                    "content_id": document.content_id or "",
+                    "meta_data": meta_data,  # type: ignore[dict-item]
+                    "content": cleaned_content,
+                    "usage": document.usage,  # type: ignore[dict-item]
+                    "content_hash": content_hash,
+                }
+                self.client.upsert(
+                    collection_name=self.collection,
+                    data=data,
+                )
+                log_debug(f"Upserted document: {document.name} ({document.meta_data})")
 
     async def async_upsert(
         self, content_hash: str, documents: List[Document], filters: Optional[Dict[str, Any]] = None
@@ -642,28 +629,46 @@ class Milvus(VectorDb):
             embed_tasks = [document.async_embed(embedder=self.embedder) for document in documents]
             await asyncio.gather(*embed_tasks, return_exceptions=True)
 
-        async def process_document(document):
-            cleaned_content = document.content.replace("\x00", "\ufffd")
-            doc_id = md5(cleaned_content.encode()).hexdigest()
-            data = {
-                "id": doc_id,
-                "vector": document.embedding,
-                "name": document.name,
-                "content_id": document.content_id,
-                "meta_data": document.meta_data,
-                "content": cleaned_content,
-                "usage": document.usage,
-                "content_hash": content_hash,
-            }
-            await self.async_client.upsert(
-                collection_name=self.collection,
-                data=data,
-            )
-            log_debug(f"Upserted document asynchronously: {document.name} ({document.meta_data})")
-            return data
+        if self.search_type == SearchType.hybrid:
+
+            async def process_hybrid_document(document):
+                data = self._prepare_document_data(content_hash=content_hash, document=document, include_vectors=True)
+                await self.async_client.upsert(
+                    collection_name=self.collection,
+                    data=data,
+                )
+                log_debug(f"Upserted hybrid document asynchronously: {document.name} ({document.meta_data})")
+                return data
+
+            await asyncio.gather(*[process_hybrid_document(doc) for doc in documents])
+        else:
+
+            async def process_document(document):
+                cleaned_content = document.content.replace("\x00", "\ufffd")
+                doc_id = md5(cleaned_content.encode()).hexdigest()
+
+                meta_data = document.meta_data or {}
+                if filters:
+                    meta_data.update(filters)
+
+                data = {
+                    "id": doc_id,
+                    "vector": document.embedding,
+                    "name": document.name,
+                    "content_id": document.content_id or "",
+                    "meta_data": meta_data,  # type: ignore[dict-item]
+                    "content": cleaned_content,
+                    "usage": document.usage,  # type: ignore[dict-item]
+                    "content_hash": content_hash,
+                }
+                await self.async_client.upsert(
+                    collection_name=self.collection,
+                    data=data,
+                )
+                log_debug(f"Upserted document asynchronously: {document.name} ({document.meta_data})")
+                return data
 
-        # Process all documents in parallel
-        await asyncio.gather(*[process_document(doc) for doc in documents])
+            await asyncio.gather(*[process_document(doc) for doc in documents])
 
         log_debug(f"Upserted {len(documents)} documents asynchronously in parallel")
 
@@ -677,7 +682,11 @@ class Milvus(VectorDb):
         return MILVUS_DISTANCE_MAP.get(self.distance, "COSINE")
 
     def search(
-        self, query: str, limit: int = 5, filters: Optional[Union[Dict[str, Any], List[FilterExpr]]] = None
+        self,
+        query: str,
+        limit: int = 5,
+        filters: Optional[Union[Dict[str, Any], List[FilterExpr]]] = None,
+        search_params: Optional[Dict[str, Any]] = None,
     ) -> List[Document]:
         """
         Search for documents matching the query.
@@ -686,6 +695,10 @@ class Milvus(VectorDb):
             query (str): Query string to search for
             limit (int): Maximum number of results to return
             filters (Optional[Dict[str, Any]]): Filters to apply to the search
+            search_params (Optional[Dict[str, Any]]): Milvus search parameters including:
+                - radius (float): Minimum similarity threshold for range search
+                - range_filter (float): Maximum similarity threshold for range search
+                - params (dict): Index-specific search params (e.g., nprobe, ef)
 
         Returns:
             List[Document]: List of matching documents
@@ -707,6 +720,7 @@ class Milvus(VectorDb):
             filter=self._build_expr(filters),
             output_fields=["*"],
             limit=limit,
+            search_params=search_params,
         )
 
         # Build search results
@@ -734,8 +748,27 @@ class Milvus(VectorDb):
         return search_results
 
     async def async_search(
-        self, query: str, limit: int = 5, filters: Optional[Union[Dict[str, Any], List[FilterExpr]]] = None
+        self,
+        query: str,
+        limit: int = 5,
+        filters: Optional[Union[Dict[str, Any], List[FilterExpr]]] = None,
+        search_params: Optional[Dict[str, Any]] = None,
     ) -> List[Document]:
+        """
+        Asynchronously search for documents matching the query.
+
+        Args:
+            query (str): Query string to search for
+            limit (int): Maximum number of results to return
+            filters (Optional[Dict[str, Any]]): Filters to apply to the search
+            search_params (Optional[Dict[str, Any]]): Milvus search parameters including:
+                - radius (float): Minimum similarity threshold for range search
+                - range_filter (float): Maximum similarity threshold for range search
+                - params (dict): Index-specific search params (e.g., nprobe, ef)
+
+        Returns:
+            List[Document]: List of matching documents
+        """
         if isinstance(filters, List):
             log_warning("Filters Expressions are not supported in Milvus. No filters will be applied.")
             filters = None
@@ -753,6 +786,7 @@ class Milvus(VectorDb):
             filter=self._build_expr(filters),
             output_fields=["*"],
             limit=limit,
+            search_params=search_params,
         )
 
         # Build search results
@@ -1099,7 +1133,7 @@ class Milvus(VectorDb):
             if isinstance(v, (list, tuple)):
                 # For array values, use json_contains_any
                 values_str = json.dumps(v)
-                expr = f'json_contains_any(meta_data, {values_str}, "{k}")'
+                expr = f'json_contains_any(meta_data["{k}"], {values_str})'
             elif isinstance(v, str):
                 # For string values
                 expr = f'meta_data["{k}"] == "{v}"'

agno/vectordb/mongodb/__init__.py

@@ -1,4 +1,4 @@
-from agno.vectordb.mongodb.mongodb import MongoDb
+from agno.vectordb.mongodb.mongodb import MongoDb, SearchType
 
 # Alias to avoid name collision with the main MongoDb class
 MongoVectorDb = MongoDb
@@ -6,4 +6,5 @@ MongoVectorDb = MongoDb
 __all__ = [
     "MongoVectorDb",
     "MongoDb",
+    "SearchType",
 ]

agno/vectordb/mongodb/mongodb.py

@@ -1,5 +1,6 @@
 import asyncio
 import time
+from importlib import metadata
 from typing import Any, Dict, List, Optional, Union
 
 from bson import ObjectId
@@ -20,11 +21,14 @@ except ImportError:
 try:
     from pymongo import AsyncMongoClient, MongoClient, errors
     from pymongo.collection import Collection
+    from pymongo.driver_info import DriverInfo
     from pymongo.operations import SearchIndexModel
 
 except ImportError:
     raise ImportError("`pymongo` not installed. Please install using `pip install pymongo`")
 
+DRIVER_METADATA = DriverInfo(name="Agno", version=metadata.version("agno"))
+
 
 class MongoDb(VectorDb):
     """
@@ -110,7 +114,7 @@ class MongoDb(VectorDb):
             from agno.knowledge.embedder.openai import OpenAIEmbedder
 
             embedder = OpenAIEmbedder()
-            log_info("Embedder not provided, using OpenAIEmbedder as default.")
+            log_debug("Embedder not provided, using OpenAIEmbedder as default.")
         self.embedder = embedder
 
         self.distance_metric = distance_metric
@@ -135,6 +139,11 @@ class MongoDb(VectorDb):
         self._async_db = None
         self._async_collection: Optional[Collection] = None
 
+        if self._client is not None:
+            # append_metadata was added in PyMongo 4.14.0, but is a valid database name on earlier versions
+            if callable(self._client.append_metadata):
+                self._client.append_metadata(DRIVER_METADATA)
+
     def _get_client(self) -> MongoClient:
         """Create or retrieve the MongoDB client."""
         if self._client is None:
@@ -157,7 +166,7 @@ class MongoDb(VectorDb):
                         warnings.filterwarnings(
                             "ignore", category=UserWarning, message=".*connected to a CosmosDB cluster.*"
                         )
-                        self._client = MongoClient(self.connection_string, **cosmos_kwargs)  # type: ignore
+                        self._client = MongoClient(self.connection_string, **cosmos_kwargs, driver=DRIVER_METADATA)  # type: ignore
 
                         self._client.admin.command("ping")
 
@@ -173,7 +182,7 @@ class MongoDb(VectorDb):
             else:
                 try:
                     log_debug("Creating MongoDB Client")
-                    self._client = MongoClient(self.connection_string, **self.kwargs)
+                    self._client = MongoClient(self.connection_string, **self.kwargs, driver=DRIVER_METADATA)  # type: ignore
                     # Trigger a connection to verify the client
                     self._client.admin.command("ping")
                     log_info("Connected to MongoDB successfully.")
@@ -195,6 +204,7 @@ class MongoDb(VectorDb):
                 maxPoolSize=self.kwargs.get("maxPoolSize", 100),
                 retryWrites=self.kwargs.get("retryWrites", True),
                 serverSelectionTimeoutMS=5000,
+                driver=DRIVER_METADATA,
             )
             # Verify connection
             try:
@@ -471,20 +481,6 @@ class MongoDb(VectorDb):
             if self.wait_until_index_ready_in_seconds:
                 await self._wait_for_index_ready_async()
 
-    def doc_exists(self, document: Document) -> bool:
-        """Check if a document exists in the MongoDB collection based on its content."""
-        try:
-            collection = self._get_collection()
-            # Use content hash as document ID
-            doc_id = md5(document.content.encode("utf-8")).hexdigest()
-            result = collection.find_one({"_id": doc_id})
-            exists = result is not None
-            log_debug(f"Document {'exists' if exists else 'does not exist'}: {doc_id}")
-            return exists
-        except Exception as e:
-            logger.error(f"Error checking document existence: {e}")
-            return False
-
     def name_exists(self, name: str) -> bool:
         """Check if a document with a given name exists in the collection."""
         try:
@@ -1024,19 +1020,6 @@ class MongoDb(VectorDb):
             logger.error(f"Error getting document count: {e}")
             return 0
 
-    async def async_doc_exists(self, document: Document) -> bool:
-        """Check if a document exists asynchronously."""
-        try:
-            collection = await self._get_async_collection()
-            doc_id = md5(document.content.encode("utf-8")).hexdigest()
-            result = await collection.find_one({"_id": doc_id})
-            exists = result is not None
-            log_debug(f"Document {'exists' if exists else 'does not exist'}: {doc_id}")
-            return exists
-        except Exception as e:
-            logger.error(f"Error checking document existence asynchronously: {e}")
-            return False
-
     async def async_insert(
         self, content_hash: str, documents: List[Document], filters: Optional[Dict[str, Any]] = None
     ) -> None:
@@ -1167,7 +1150,7 @@ class MongoDb(VectorDb):
         if isinstance(filters, List):
             log_warning("Filters Expressions are not supported in MongoDB. No filters will be applied.")
             filters = None
-        query_embedding = self.embedder.get_embedding(query)
+        query_embedding = await self.embedder.async_get_embedding(query)
         if query_embedding is None:
             logger.error(f"Failed to generate embedding for query: {query}")
             return []