agno 1.8.2__py3-none-any.whl → 2.0.0__py3-none-any.whl

agno/vectordb/lancedb/lance_db.py

@@ -1,3 +1,4 @@
+import asyncio
 import json
 from hashlib import md5
 from os import getenv
@@ -9,9 +10,9 @@ try:
 except ImportError:
     raise ImportError("`lancedb` not installed. Please install using `pip install lancedb`")
 
-from agno.document import Document
-from agno.embedder import Embedder
-from agno.reranker.base import Reranker
+from agno.knowledge.document import Document
+from agno.knowledge.embedder import Embedder
+from agno.knowledge.reranker.base import Reranker
 from agno.utils.log import log_debug, log_info, logger
 from agno.vectordb.base import VectorDb
 from agno.vectordb.distance import Distance
@@ -60,7 +61,7 @@ class LanceDb(VectorDb):
     ):
         # Embedder for embedding the document contents
         if embedder is None:
-            from agno.embedder.openai import OpenAIEmbedder
+            from agno.knowledge.embedder.openai import OpenAIEmbedder
 
             embedder = OpenAIEmbedder()
             log_info("Embedder not provided, using OpenAIEmbedder as default.")
@@ -88,10 +89,18 @@ class LanceDb(VectorDb):
 
         if table_name and table_name in self.connection.table_names():
             # Open the table if it exists
-            self.table = self.connection.open_table(name=table_name)
-            self.table_name = self.table.name
-            self._vector_col = self.table.schema.names[0]
-            self._id = self.table.schema.names[1]  # type: ignore
+            try:
+                self.table = self.connection.open_table(name=table_name)
+                self.table_name = self.table.name
+                self._vector_col = self.table.schema.names[0]
+                self._id = self.table.schema.names[1]  # type: ignore
+            except ValueError as e:
+                # Table might have been dropped by async operations but sync connection hasn't updated
+                if "was not found" in str(e):
+                    log_debug(f"Table {table_name} listed but not accessible, will create if needed")
+                    self.table = None
+                else:
+                    raise
 
         # LanceDB table details
         if self.table is None:
@@ -105,7 +114,7 @@ class LanceDb(VectorDb):
                 self.table = table
                 self.table_name = self.table.name
                 self._vector_col = self.table.schema.names[0]
-                self._id = self.tbl.schema.names[1]  # type: ignore
+                self._id = self.table.schema.names[1]  # type: ignore
             else:
                 if not table_name:
                     raise ValueError("Either table or table_name should be provided.")
@@ -135,10 +144,28 @@ class LanceDb(VectorDb):
         """Get or create an async connection to LanceDB."""
         if self.async_connection is None:
             self.async_connection = await lancedb.connect_async(self.uri)
+        # Only try to open table if it exists and we don't have it already
         if self.async_table is None:
-            self.async_table = await self.async_connection.open_table(self.table_name)
+            table_names = await self.async_connection.table_names()
+            if self.table_name in table_names:
+                try:
+                    self.async_table = await self.async_connection.open_table(self.table_name)
+                except ValueError:
+                    # Table might have been dropped by another operation
+                    pass
         return self.async_connection
 
+    def _refresh_sync_connection(self) -> None:
+        """Refresh the sync connection to see changes made by async operations."""
+        try:
+            # Re-establish sync connection to see async changes
+            if self.connection and self.table_name in self.connection.table_names():
+                self.table = self.connection.open_table(self.table_name)
+                log_debug(f"Refreshed sync connection for table: {self.table_name}")
+        except Exception as e:
+            log_debug(f"Could not refresh sync connection: {e}")
+            # If refresh fails, we can still function but sync methods might not see async changes
+
     def create(self) -> None:
         """Create the table if it does not exist."""
         if not self.exists():
@@ -146,7 +173,7 @@ class LanceDb(VectorDb):
 
     async def async_create(self) -> None:
         """Create the table asynchronously if it does not exist."""
-        if not self.exists():
+        if not await self.async_exists():
             conn = await self._get_async_connection()
             schema = self._base_schema()
 
@@ -212,7 +239,7 @@ class LanceDb(VectorDb):
             self.table = self.connection.open_table(name=self.table_name)
         return self.doc_exists(document)
 
-    def insert(self, documents: List[Document], filters: Optional[Dict[str, Any]] = None) -> None:
+    def insert(self, content_hash: str, documents: List[Document], filters: Optional[Dict[str, Any]] = None) -> None:
         """
         Insert documents into the database.
 
@@ -245,6 +272,8 @@ class LanceDb(VectorDb):
                 "meta_data": document.meta_data,
                 "content": cleaned_content,
                 "usage": document.usage,
+                "content_id": document.content_id,
+                "content_hash": content_hash,
             }
             data.append(
                 {
@@ -270,7 +299,9 @@ class LanceDb(VectorDb):
 
         log_debug(f"Inserted {len(data)} documents")
 
-    async def async_insert(self, documents: List[Document], filters: Optional[Dict[str, Any]] = None) -> None:
+    async def async_insert(
+        self, content_hash: str, documents: List[Document], filters: Optional[Dict[str, Any]] = None
+    ) -> None:
         """
         Asynchronously insert documents into the database.
 
@@ -285,7 +316,10 @@ class LanceDb(VectorDb):
         log_debug(f"Inserting {len(documents)} documents")
         data = []
 
-        # Prepare documents for insertion
+        # Prepare documents for insertion.
+        embed_tasks = [document.async_embed(embedder=self.embedder) for document in documents]
+        await asyncio.gather(*embed_tasks, return_exceptions=True)
+
         for document in documents:
             if await self.async_doc_exists(document):
                 continue
@@ -296,7 +330,6 @@ class LanceDb(VectorDb):
                 meta_data.update(filters)
                 document.meta_data = meta_data
 
-            document.embed(embedder=self.embedder)
             cleaned_content = document.content.replace("\x00", "\ufffd")
             doc_id = str(md5(cleaned_content.encode()).hexdigest())
             payload = {
@@ -304,6 +337,8 @@ class LanceDb(VectorDb):
                 "meta_data": document.meta_data,
                 "content": cleaned_content,
                 "usage": document.usage,
+                "content_id": document.content_id,
+                "content_hash": content_hash,
             }
             data.append(
                 {
@@ -327,11 +362,18 @@ class LanceDb(VectorDb):
                 await self.async_table.add(data)  # type: ignore
 
             log_debug(f"Asynchronously inserted {len(data)} documents")
+
+            # Refresh sync connection to see async changes
+            self._refresh_sync_connection()
         except Exception as e:
             logger.error(f"Error during async document insertion: {e}")
             raise
 
-    def upsert(self, documents: List[Document], filters: Optional[Dict[str, Any]] = None) -> None:
+    def upsert_available(self) -> bool:
+        """Check if upsert is available in LanceDB."""
+        return True
+
+    def upsert(self, content_hash: str, documents: List[Document], filters: Optional[Dict[str, Any]] = None) -> None:
         """
         Upsert documents into the database.
 
@@ -339,10 +381,16 @@ class LanceDb(VectorDb):
             documents (List[Document]): List of documents to upsert
             filters (Optional[Dict[str, Any]]): Filters to apply while upserting
         """
-        self.insert(documents, filters=filters)
+        if self.content_hash_exists(content_hash):
+            self._delete_by_content_hash(content_hash)
+        self.insert(content_hash=content_hash, documents=documents, filters=filters)
 
-    async def async_upsert(self, documents: List[Document], filters: Optional[Dict[str, Any]] = None) -> None:
-        await self.async_insert(documents, filters)
+    async def async_upsert(
+        self, content_hash: str, documents: List[Document], filters: Optional[Dict[str, Any]] = None
+    ) -> None:
+        if self.content_hash_exists(content_hash):
+            self._delete_by_content_hash(content_hash)
+        await self.async_insert(content_hash=content_hash, documents=documents, filters=filters)
 
     def search(self, query: str, limit: int = 5, filters: Optional[Dict[str, Any]] = None) -> List[Document]:
         """
@@ -539,6 +587,7 @@ class LanceDb(VectorDb):
                         embedder=self.embedder,
                         embedding=item["vector"],
                         usage=payload["usage"],
+                        content_id=payload.get("content_id"),
                     )
                 )
 
@@ -551,6 +600,8 @@ class LanceDb(VectorDb):
         if self.exists():
             log_debug(f"Deleting collection: {self.table_name}")
             self.connection.drop_table(self.table_name)  # type: ignore
+            # Clear the table reference after dropping
+            self.table = None
 
     async def async_drop(self) -> None:
         """Drop the table asynchronously."""
@@ -558,16 +609,26 @@ class LanceDb(VectorDb):
             log_debug(f"Deleting collection: {self.table_name}")
             conn = await self._get_async_connection()
             await conn.drop_table(self.table_name)
+            # Clear the async table reference after dropping
+            self.async_table = None
 
     def exists(self) -> bool:
+        # If we have an async table that was created, the table exists
+        if self.async_table is not None:
+            return True
         if self.connection:
             return self.table_name in self.connection.table_names()
         return False
 
     async def async_exists(self) -> bool:
         """Check if the table exists asynchronously."""
-        conn = await self._get_async_connection()
-        table_names = await conn.table_names()
+        # If we have an async table that was created, the table exists
+        if self.async_table is not None:
+            return True
+        # Check if table exists in database without trying to open it
+        if self.async_connection is None:
+            self.async_connection = await lancedb.connect_async(self.uri)
+        table_names = await self.async_connection.table_names()
         return self.table_name in table_names
 
     async def async_get_count(self) -> int:
@@ -577,7 +638,27 @@ class LanceDb(VectorDb):
             return await self.async_table.count_rows()
         return 0
 
+    def _async_get_count_sync(self) -> int:
+        """Helper method to run async_get_count in a new thread with its own event loop"""
+        import asyncio
+
+        return asyncio.run(self.async_get_count())
+
     def get_count(self) -> int:
+        # If we have data in the async table but sync table isn't available, try to get count from async table
+        if self.async_table is not None:
+            try:
+                import asyncio
+
+                # Check if we're already in an async context
+                try:
+                    return self._async_get_count_sync()
+                except RuntimeError:
+                    # No event loop running, safe to use asyncio.run
+                    return asyncio.run(self.async_get_count())
+            except Exception:
+                pass
+
         if self.exists() and self.table:
             return self.table.count_rows()
         return 0
@@ -606,3 +687,261 @@ class LanceDb(VectorDb):
 
     async def async_name_exists(self, name: str) -> bool:
         raise NotImplementedError(f"Async not supported on {self.__class__.__name__}.")
+
+    def id_exists(self, id: str) -> bool:
+        """Check if a document with the given ID exists in the database"""
+        if self.table is None:
+            logger.error("Table not initialized")
+            return False
+
+        try:
+            # Search for the document with the specific ID
+            result = self.table.search().where(f"{self._id} = '{id}'").to_pandas()
+            return len(result) > 0
+        except Exception as e:
+            logger.error(f"Error checking id existence: {e}")
+            return False
+
+    def delete_by_id(self, id: str) -> bool:
+        """Delete content by ID."""
+        if self.table is None:
+            logger.error("Table not initialized")
+            return False
+
+        try:
+            # Delete rows where the id matches
+            self.table.delete(f"{self._id} = '{id}'")
+            log_info(f"Deleted records with id '{id}' from table '{self.table_name}'.")
+            return True
+        except Exception as e:
+            logger.error(f"Error deleting rows by id '{id}': {e}")
+            return False
+
+    def delete_by_name(self, name: str) -> bool:
+        """Delete content by name."""
+        if self.table is None:
+            logger.error("Table not initialized")
+            return False
+
+        try:
+            total_count = self.table.count_rows()
+            result = self.table.search().select(["id", "payload"]).limit(total_count).to_pandas()
+
+            # Find matching IDs
+            ids_to_delete = []
+            for _, row in result.iterrows():
+                payload = json.loads(row["payload"])
+                if payload.get("name") == name:
+                    ids_to_delete.append(row["id"])
+
+            # Delete matching records
+            if ids_to_delete:
+                for doc_id in ids_to_delete:
+                    self.table.delete(f"{self._id} = '{doc_id}'")
+                log_info(f"Deleted {len(ids_to_delete)} records with name '{name}' from table '{self.table_name}'.")
+                return True
+            else:
+                log_info(f"No records found with name '{name}' to delete.")
+                return False
+
+        except Exception as e:
+            logger.error(f"Error deleting rows by name '{name}': {e}")
+            return False
+
+    def delete_by_metadata(self, metadata: Dict[str, Any]) -> bool:
+        """Delete content by metadata."""
+        if self.table is None:
+            logger.error("Table not initialized")
+            return False
+
+        try:
+            total_count = self.table.count_rows()
+            result = self.table.search().select(["id", "payload"]).limit(total_count).to_pandas()
+
+            # Find matching IDs
+            ids_to_delete = []
+            for _, row in result.iterrows():
+                payload = json.loads(row["payload"])
+                doc_metadata = payload.get("meta_data", {})
+
+                # Check if all metadata key-value pairs match
+                match = True
+                for key, value in metadata.items():
+                    if key not in doc_metadata or doc_metadata[key] != value:
+                        match = False
+                        break
+
+                if match:
+                    ids_to_delete.append(row["id"])
+
+            # Delete matching records
+            if ids_to_delete:
+                for doc_id in ids_to_delete:
+                    self.table.delete(f"{self._id} = '{doc_id}'")
+                log_info(
+                    f"Deleted {len(ids_to_delete)} records with metadata '{metadata}' from table '{self.table_name}'."
+                )
+                return True
+            else:
+                log_info(f"No records found with metadata '{metadata}' to delete.")
+                return False
+
+        except Exception as e:
+            logger.error(f"Error deleting rows by metadata '{metadata}': {e}")
+            return False
+
+    def delete_by_content_id(self, content_id: str) -> bool:
+        """Delete content by content ID."""
+        if self.table is None:
+            logger.error("Table not initialized")
+            return False
+
+        try:
+            total_count = self.table.count_rows()
+            result = self.table.search().select(["id", "payload"]).limit(total_count).to_pandas()
+
+            # Find matching IDs
+            ids_to_delete = []
+            for _, row in result.iterrows():
+                payload = json.loads(row["payload"])
+                if payload.get("content_id") == content_id:
+                    ids_to_delete.append(row["id"])
+
+            # Delete matching records
+            if ids_to_delete:
+                for doc_id in ids_to_delete:
+                    self.table.delete(f"{self._id} = '{doc_id}'")
+                log_info(
+                    f"Deleted {len(ids_to_delete)} records with content_id '{content_id}' from table '{self.table_name}'."
+                )
+                return True
+            else:
+                log_info(f"No records found with content_id '{content_id}' to delete.")
+                return False
+
+        except Exception as e:
+            logger.error(f"Error deleting rows by content_id '{content_id}': {e}")
+            return False
+
+    def _delete_by_content_hash(self, content_hash: str) -> bool:
+        """Delete content by content hash."""
+        if self.table is None:
+            logger.error("Table not initialized")
+            return False
+
+        try:
+            total_count = self.table.count_rows()
+            result = self.table.search().select(["id", "payload"]).limit(total_count).to_pandas()
+
+            # Find matching IDs
+            ids_to_delete = []
+            for _, row in result.iterrows():
+                payload = json.loads(row["payload"])
+                if payload.get("content_hash") == content_hash:
+                    ids_to_delete.append(row["id"])
+
+            # Delete matching records
+            if ids_to_delete:
+                for doc_id in ids_to_delete:
+                    self.table.delete(f"{self._id} = '{doc_id}'")
+                log_info(
+                    f"Deleted {len(ids_to_delete)} records with content_hash '{content_hash}' from table '{self.table_name}'."
+                )
+                return True
+            else:
+                log_info(f"No records found with content_hash '{content_hash}' to delete.")
+                return False
+
+        except Exception as e:
+            logger.error(f"Error deleting rows by content_hash '{content_hash}': {e}")
+            return False
+
+    def content_hash_exists(self, content_hash: str) -> bool:
+        """Check if documents with the given content hash exist."""
+        if self.table is None:
+            logger.error("Table not initialized")
+            return False
+
+        try:
+            total_count = self.table.count_rows()
+            result = self.table.search().select(["id", "payload"]).limit(total_count).to_pandas()
+
+            # Check if any records match the content_hash
+            for _, row in result.iterrows():
+                payload = json.loads(row["payload"])
+                if payload.get("content_hash") == content_hash:
+                    return True
+
+            return False
+
+        except Exception as e:
+            logger.error(f"Error checking content_hash existence '{content_hash}': {e}")
+            return False
+
+    def update_metadata(self, content_id: str, metadata: Dict[str, Any]) -> None:
+        """
+        Update the metadata for documents with the given content_id.
+
+        Args:
+            content_id (str): The content ID to update
+            metadata (Dict[str, Any]): The metadata to update
+        """
+        import json
+
+        try:
+            if self.table is None:
+                logger.error("Table not initialized")
+                return
+
+            # Search for documents with the given content_id
+            query_filter = f"payload->>'content_id' = '{content_id}'"
+            results = self.table.search().where(query_filter).to_pandas()
+
+            if results.empty:
+                logger.debug(f"No documents found with content_id: {content_id}")
+                return
+
+            # Update each matching document
+            updated_count = 0
+            for _, row in results.iterrows():
+                row_id = row["id"]
+                current_payload = json.loads(row["payload"])
+
+                # Merge existing metadata with new metadata
+                if "meta_data" in current_payload:
+                    current_payload["meta_data"].update(metadata)
+                else:
+                    current_payload["meta_data"] = metadata
+
+                if "filters" in current_payload:
+                    if isinstance(current_payload["filters"], dict):
+                        current_payload["filters"].update(metadata)
+                    else:
+                        current_payload["filters"] = metadata
+                else:
+                    current_payload["filters"] = metadata
+
+                # Update the document
+                update_data = {"id": row_id, "payload": json.dumps(current_payload)}
+
+                # LanceDB doesn't have a direct update, so we need to delete and re-insert
+                # First, get all the existing data
+                vector_data = row["vector"] if "vector" in row else None
+                text_data = row["text"] if "text" in row else None
+
+                # Create complete update record
+                if vector_data is not None:
+                    update_data["vector"] = vector_data
+                if text_data is not None:
+                    update_data["text"] = text_data
+
+                # Delete old record and insert updated one
+                self.table.delete(f"id = '{row_id}'")
+                self.table.add([update_data])
+                updated_count += 1
+
+            logger.debug(f"Updated metadata for {updated_count} documents with content_id: {content_id}")
+
+        except Exception as e:
+            logger.error(f"Error updating metadata for content_id '{content_id}': {e}")
+            raise

agno/vectordb/langchaindb/__init__.py

@@ -0,0 +1,5 @@
+from agno.vectordb.langchaindb.langchaindb import LangChainVectorDb
+
+__all__ = [
+    "LangChainVectorDb",
+]

agno/vectordb/langchaindb/langchaindb.py

@@ -0,0 +1,145 @@
+from typing import Any, Dict, List, Optional
+
+from agno.knowledge.document import Document
+from agno.utils.log import log_debug, logger
+from agno.vectordb.base import VectorDb
+
+
+class LangChainVectorDb(VectorDb):
+    def __init__(
+        self,
+        vectorstore: Optional[Any] = None,
+        search_kwargs: Optional[dict] = None,
+        knowledge_retriever: Optional[Any] = None,
+    ):
+        """
+        Initialize LangChainVectorDb.
+
+        Args:
+            vectorstore: The LangChain vectorstore instance
+            search_kwargs: Additional search parameters for the retriever
+            knowledge_retriever: An optional LangChain retriever instance
+        """
+        self.vectorstore = vectorstore
+        self.search_kwargs = search_kwargs
+        self.knowledge_retriever = knowledge_retriever
+
+    def create(self) -> None:
+        raise NotImplementedError
+
+    async def async_create(self) -> None:
+        raise NotImplementedError
+
+    def name_exists(self, name: str) -> bool:
+        raise NotImplementedError
+
+    def async_name_exists(self, name: str) -> bool:
+        raise NotImplementedError
+
+    def id_exists(self, id: str) -> bool:
+        raise NotImplementedError
+
+    def content_hash_exists(self, content_hash: str) -> bool:
+        raise NotImplementedError
+
+    def delete_by_content_id(self, content_id: str) -> None:
+        raise NotImplementedError
+
+    def insert(self, content_hash: str, documents: List[Document], filters: Optional[Dict[str, Any]] = None) -> None:
+        logger.warning("LangChainKnowledgeBase.insert() not supported - please check the vectorstore manually.")
+        raise NotImplementedError
+
+    async def async_insert(
+        self, content_hash: str, documents: List[Document], filters: Optional[Dict[str, Any]] = None
+    ) -> None:
+        logger.warning("LangChainKnowledgeBase.async_insert() not supported - please check the vectorstore manually.")
+        raise NotImplementedError
+
+    def upsert(self, content_hash: str, documents: List[Document], filters: Optional[Dict[str, Any]] = None) -> None:
+        logger.warning("LangChainKnowledgeBase.upsert() not supported - please check the vectorstore manually.")
+        raise NotImplementedError
+
+    async def async_upsert(self, documents: List[Document], filters: Optional[Dict[str, Any]] = None) -> None:
+        logger.warning("LangChainKnowledgeBase.async_upsert() not supported - please check the vectorstore manually.")
+        raise NotImplementedError
+
+    def search(
+        self, query: str, num_documents: Optional[int] = None, filters: Optional[Dict[str, Any]] = None
+    ) -> List[Document]:
+        """Returns relevant documents matching the query"""
+
+        try:
+            from langchain_core.documents import Document as LangChainDocument
+            from langchain_core.retrievers import BaseRetriever
+        except ImportError:
+            raise ImportError(
+                "The `langchain` package is not installed. Please install it via `pip install langchain`."
+            )
+
+        if self.vectorstore is not None and self.knowledge_retriever is None:
+            log_debug("Creating knowledge retriever")
+            if self.search_kwargs is None:
+                self.search_kwargs = {"k": num_documents}
+            if filters is not None:
+                self.search_kwargs.update(filters)
+            self.knowledge_retriever = self.vectorstore.as_retriever(search_kwargs=self.search_kwargs)
+
+        if self.knowledge_retriever is None:
+            logger.error("No knowledge retriever provided")
+            return []
+
+        if not isinstance(self.knowledge_retriever, BaseRetriever):
+            raise ValueError(f"Knowledge retriever is not of type BaseRetriever: {self.knowledge_retriever}")
+
+        log_debug(f"Getting {num_documents} relevant documents for query: {query}")
+        lc_documents: List[LangChainDocument] = self.knowledge_retriever.invoke(input=query)
+        documents = []
+        for lc_doc in lc_documents:
+            documents.append(
+                Document(
+                    content=lc_doc.page_content,
+                    meta_data=lc_doc.metadata,
+                )
+            )
+        return documents
+
+    async def async_search(
+        self, query: str, num_documents: Optional[int] = None, filters: Optional[Dict[str, Any]] = None
+    ) -> List[Document]:
+        return self.search(query, num_documents, filters)
+
+    def drop(self) -> None:
+        raise NotImplementedError
+
+    async def async_drop(self) -> None:
+        raise NotImplementedError
+
+    async def async_exists(self) -> bool:
+        raise NotImplementedError
+
+    def delete(self) -> bool:
+        raise NotImplementedError
+
+    def delete_by_id(self, id: str) -> bool:
+        raise NotImplementedError
+
+    def delete_by_name(self, name: str) -> bool:
+        raise NotImplementedError
+
+    def delete_by_metadata(self, metadata: Dict[str, Any]) -> bool:
+        raise NotImplementedError
+
+    def exists(self) -> bool:
+        logger.warning("LangChainKnowledgeBase.exists() not supported - please check the vectorstore manually.")
+        return True
+
+    def update_metadata(self, content_id: str, metadata: Dict[str, Any]) -> None:
+        """
+        Update the metadata for documents with the given content_id.
+        Not implemented for LangChain wrapper.
+
+        Args:
+            content_id (str): The content ID to update
+            metadata (Dict[str, Any]): The metadata to update
+        """
+        raise NotImplementedError("update_metadata not supported for LangChain vectorstores")

agno/vectordb/lightrag/__init__.py

@@ -0,0 +1,5 @@
+from agno.vectordb.lightrag.lightrag import LightRag
+
+__all__ = [
+    "LightRag",
+]