agno 1.8.1__py3-none-any.whl → 2.0.0a1__py3-none-any.whl

agno/vectordb/qdrant/qdrant.py

@@ -9,10 +9,10 @@ except ImportError:
         "The `qdrant-client` package is not installed. Please install it via `pip install qdrant-client`."
     )
 
-from agno.document import Document
-from agno.embedder import Embedder
-from agno.reranker.base import Reranker
-from agno.utils.log import log_debug, log_info
+from agno.knowledge.document import Document
+from agno.knowledge.embedder import Embedder
+from agno.knowledge.reranker.base import Reranker
+from agno.utils.log import log_debug, log_error, log_info, log_warning
 from agno.vectordb.base import VectorDb
 from agno.vectordb.distance import Distance
 from agno.vectordb.search import SearchType
@@ -78,7 +78,7 @@ class Qdrant(VectorDb):
 
         # Embedder for embedding the document contents
         if embedder is None:
-            from agno.embedder.openai import OpenAIEmbedder
+            from agno.knowledge.embedder.openai import OpenAIEmbedder
 
             embedder = OpenAIEmbedder()
             log_info("Embedder not provided, using OpenAIEmbedder as default.")
@@ -302,7 +302,13 @@ class Qdrant(VectorDb):
             return len(scroll_result[0]) > 0
         return False
 
-    def insert(self, documents: List[Document], filters: Optional[Dict[str, Any]] = None, batch_size: int = 10) -> None:
+    def insert(
+        self,
+        content_hash: str,
+        documents: List[Document],
+        filters: Optional[Dict[str, Any]] = None,
+        batch_size: int = 10,
+    ) -> None:
         """
         Insert documents into the database.
 
@@ -343,6 +349,8 @@ class Qdrant(VectorDb):
                 "meta_data": document.meta_data,
                 "content": cleaned_content,
                 "usage": document.usage,
+                "content_id": document.content_id,
+                "content_hash": content_hash,
             }
 
             # Add filters as metadata if provided
@@ -364,7 +372,9 @@ class Qdrant(VectorDb):
             self.client.upsert(collection_name=self.collection, wait=False, points=points)
         log_debug(f"Upsert {len(points)} documents")
 
-    async def async_insert(self, documents: List[Document], filters: Optional[Dict[str, Any]] = None) -> None:
+    async def async_insert(
+        self, content_hash: str, documents: List[Document], filters: Optional[Dict[str, Any]] = None
+    ) -> None:
         """
         Insert documents asynchronously.
 
@@ -401,6 +411,8 @@ class Qdrant(VectorDb):
                 "meta_data": document.meta_data,
                 "content": cleaned_content,
                 "usage": document.usage,
+                "content_id": document.content_id,
+                "content_hash": content_hash,
             }
 
             # Add filters as metadata if provided
@@ -426,7 +438,7 @@ class Qdrant(VectorDb):
             await self.async_client.upsert(collection_name=self.collection, wait=False, points=points)
         log_debug(f"Upserted {len(points)} documents asynchronously")
 
-    def upsert(self, documents: List[Document], filters: Optional[Dict[str, Any]] = None) -> None:
+    def upsert(self, content_hash: str, documents: List[Document], filters: Optional[Dict[str, Any]] = None) -> None:
         """
         Upsert documents into the database.
 
@@ -435,12 +447,16 @@ class Qdrant(VectorDb):
             filters (Optional[Dict[str, Any]]): Filters to apply while upserting
         """
         log_debug("Redirecting the request to insert")
-        self.insert(documents, filters)
+        if self.content_hash_exists(content_hash):
+            self._delete_by_content_hash(content_hash)
+        self.insert(content_hash=content_hash, documents=documents, filters=filters)
 
-    async def async_upsert(self, documents: List[Document], filters: Optional[Dict[str, Any]] = None) -> None:
+    async def async_upsert(
+        self, content_hash: str, documents: List[Document], filters: Optional[Dict[str, Any]] = None
+    ) -> None:
         """Upsert documents asynchronously."""
         log_debug("Redirecting the async request to async_insert")
-        await self.async_insert(documents, filters)
+        await self.async_insert(content_hash=content_hash, documents=documents, filters=filters)
 
     def search(self, query: str, limit: int = 5, filters: Optional[Dict[str, Any]] = None) -> List[Document]:
         """
@@ -641,7 +657,8 @@ class Qdrant(VectorDb):
                     content=result.payload["content"],
                     embedder=self.embedder,
                     embedding=result.vector,  # type: ignore
-                    usage=result.payload["usage"],
+                    usage=result.payload.get("usage"),
+                    content_id=result.payload.get("content_id"),
                 )
             )
 
@@ -676,6 +693,9 @@ class Qdrant(VectorDb):
 
         return None
 
+    def optimize(self) -> None:
+        pass
+
     def drop(self) -> None:
         if self.exists():
             log_debug(f"Deleting collection: {self.collection}")
@@ -699,12 +719,312 @@ class Qdrant(VectorDb):
         count_result: models.CountResult = self.client.count(collection_name=self.collection, exact=True)
         return count_result.count
 
-    def optimize(self) -> None:
-        pass
+    def point_exists(self, id: str) -> bool:
+        """Check if a point with the given ID exists in the collection."""
+        try:
+            log_info(f"Checking if point with ID '{id}' (type: {type(id)}) exists in collection '{self.collection}'")
+            points = self.client.retrieve(
+                collection_name=self.collection, ids=[id], with_payload=False, with_vectors=False
+            )
+            log_info(f"Retrieved {len(points)} points for ID '{id}'")
+            if len(points) > 0:
+                log_info(f"Found point with ID: {points[0].id} (type: {type(points[0].id)})")
+            return len(points) > 0
+        except Exception as e:
+            log_info(f"Error checking if point {id} exists: {e}")
+            return False
 
     def delete(self) -> bool:
         return self.client.delete_collection(collection_name=self.collection)
 
+    def delete_by_id(self, id: str) -> bool:
+        try:
+            # Check if point exists before deletion
+            if not self.point_exists(id):
+                log_warning(f"Point with ID {id} does not exist")
+                return True
+
+            self.client.delete(
+                collection_name=self.collection,
+                points_selector=models.PointIdsList(points=[id]),
+                wait=True,  # Wait for the operation to complete
+            )
+            return True
+
+        except Exception as e:
+            log_info(f"Error deleting point with ID {id}: {e}")
+            return False
+
+    def delete_by_name(self, name: str) -> bool:
+        """Delete all points that have the specified name in their payload (precise match)."""
+        try:
+            log_info(f"Attempting to delete all points with name: {name}")
+
+            # Create a filter to find all points with the specified name (precise match)
+            filter_condition = models.Filter(
+                must=[models.FieldCondition(key="name", match=models.MatchValue(value=name))]
+            )
+
+            # First, count how many points will be deleted
+            count_result = self.client.count(collection_name=self.collection, count_filter=filter_condition, exact=True)
+
+            if count_result.count == 0:
+                log_warning(f"No points found with name: {name}")
+                return True
+
+            log_info(f"Found {count_result.count} points to delete with name: {name}")
+
+            # Delete all points matching the filter
+            result = self.client.delete(
+                collection_name=self.collection,
+                points_selector=filter_condition,
+                wait=True,  # Wait for the operation to complete
+            )
+
+            # Check if the deletion was successful
+            if result.status == models.UpdateStatus.COMPLETED:
+                log_info(f"Successfully deleted {count_result.count} points with name: {name}")
+                return True
+            else:
+                log_warning(f"Deletion failed for name {name}. Status: {result.status}")
+                return False
+
+        except Exception as e:
+            log_warning(f"Error deleting points with name {name}: {e}")
+            return False
+
+    def delete_by_metadata(self, metadata: Dict[str, Any]) -> bool:
+        """Delete all points where the given metadata is contained in the meta_data payload field."""
+        try:
+            log_info(f"Attempting to delete all points with metadata: {metadata}")
+
+            # Create filter conditions for each metadata key-value pair
+            filter_conditions = []
+            for key, value in metadata.items():
+                # Use the meta_data prefix since that's how metadata is stored in the payload
+                filter_conditions.append(
+                    models.FieldCondition(key=f"meta_data.{key}", match=models.MatchValue(value=value))
+                )
+
+            # Create a filter that requires ALL metadata conditions to match
+            filter_condition = models.Filter(must=filter_conditions)
+
+            # First, count how many points will be deleted
+            count_result = self.client.count(collection_name=self.collection, count_filter=filter_condition, exact=True)
+
+            if count_result.count == 0:
+                log_warning(f"No points found with metadata: {metadata}")
+                return True
+
+            log_info(f"Found {count_result.count} points to delete with metadata: {metadata}")
+
+            # Delete all points matching the filter
+            result = self.client.delete(
+                collection_name=self.collection,
+                points_selector=filter_condition,
+                wait=True,  # Wait for the operation to complete
+            )
+
+            # Check if the deletion was successful
+            if result.status == models.UpdateStatus.COMPLETED:
+                log_info(f"Successfully deleted {count_result.count} points with metadata: {metadata}")
+                return True
+            else:
+                log_warning(f"Deletion failed for metadata {metadata}. Status: {result.status}")
+                return False
+
+        except Exception as e:
+            log_warning(f"Error deleting points with metadata {metadata}: {e}")
+            return False
+
+    def delete_by_content_id(self, content_id: str) -> bool:
+        """Delete all points that have the specified content_id in their payload."""
+        try:
+            log_info(f"Attempting to delete all points with content_id: {content_id}")
+
+            # Create a filter to find all points with the specified content_id
+            filter_condition = models.Filter(
+                must=[models.FieldCondition(key="content_id", match=models.MatchValue(value=content_id))]
+            )
+
+            # First, count how many points will be deleted
+            count_result = self.client.count(collection_name=self.collection, count_filter=filter_condition, exact=True)
+
+            if count_result.count == 0:
+                log_warning(f"No points found with content_id: {content_id}")
+                return True
+
+            log_info(f"Found {count_result.count} points to delete with content_id: {content_id}")
+
+            # Delete all points matching the filter
+            result = self.client.delete(
+                collection_name=self.collection,
+                points_selector=filter_condition,
+                wait=True,  # Wait for the operation to complete
+            )
+
+            # Check if the deletion was successful
+            if result.status == models.UpdateStatus.COMPLETED:
+                log_info(f"Successfully deleted {count_result.count} points with content_id: {content_id}")
+                return True
+            else:
+                log_warning(f"Deletion failed for content_id {content_id}. Status: {result.status}")
+                return False
+
+        except Exception as e:
+            log_warning(f"Error deleting points with content_id {content_id}: {e}")
+            return False
+
+    def id_exists(self, id: str) -> bool:
+        """Check if a point with the given ID exists in the collection.
+
+        Args:
+            id (str): The ID to check.
+
+        Returns:
+            bool: True if the point exists, False otherwise.
+        """
+        try:
+            points = self.client.retrieve(
+                collection_name=self.collection, ids=[id], with_payload=False, with_vectors=False
+            )
+            return len(points) > 0
+        except Exception as e:
+            log_info(f"Error checking if point {id} exists: {e}")
+            return False
+
+    def content_hash_exists(self, content_hash: str) -> bool:
+        """Check if any points with the given content hash exist in the collection.
+
+        Args:
+            content_hash (str): The content hash to check.
+
+        Returns:
+            bool: True if points with the content hash exist, False otherwise.
+        """
+        try:
+            # Create a filter to find points with the specified content_hash
+            filter_condition = models.Filter(
+                must=[models.FieldCondition(key="content_hash", match=models.MatchValue(value=content_hash))]
+            )
+
+            # Count how many points match the filter
+            count_result = self.client.count(collection_name=self.collection, count_filter=filter_condition, exact=True)
+            return count_result.count > 0
+        except Exception as e:
+            log_info(f"Error checking if content_hash {content_hash} exists: {e}")
+            return False
+
+    def _delete_by_content_hash(self, content_hash: str) -> bool:
+        """Delete all points that have the specified content_hash in their payload.
+
+        Args:
+            content_hash (str): The content hash to delete.
+
+        Returns:
+            bool: True if points were deleted successfully, False otherwise.
+        """
+        try:
+            log_info(f"Attempting to delete all points with content_hash: {content_hash}")
+
+            # Create a filter to find all points with the specified content_hash
+            filter_condition = models.Filter(
+                must=[models.FieldCondition(key="content_hash", match=models.MatchValue(value=content_hash))]
+            )
+
+            # First, count how many points will be deleted
+            count_result = self.client.count(collection_name=self.collection, count_filter=filter_condition, exact=True)
+
+            if count_result.count == 0:
+                log_warning(f"No points found with content_hash: {content_hash}")
+                return True
+
+            log_info(f"Found {count_result.count} points to delete with content_hash: {content_hash}")
+
+            # Delete all points matching the filter
+            result = self.client.delete(
+                collection_name=self.collection,
+                points_selector=filter_condition,
+                wait=True,  # Wait for the operation to complete
+            )
+
+            # Check if the deletion was successful
+            if result.status == models.UpdateStatus.COMPLETED:
+                log_info(f"Successfully deleted {count_result.count} points with content_hash: {content_hash}")
+                return True
+            else:
+                log_warning(f"Deletion failed for content_hash {content_hash}. Status: {result.status}")
+                return False
+
+        except Exception as e:
+            log_warning(f"Error deleting points with content_hash {content_hash}: {e}")
+            return False
+
+    def update_metadata(self, content_id: str, metadata: Dict[str, Any]) -> None:
+        """
+        Update the metadata for documents with the given content_id.
+
+        Args:
+            content_id (str): The content ID to update
+            metadata (Dict[str, Any]): The metadata to update
+        """
+        try:
+            if not self.client:
+                log_error("Client not initialized")
+                return
+
+            # Create filter for content_id
+            filter_condition = models.Filter(
+                must=[models.FieldCondition(key="content_id", match=models.MatchValue(value=content_id))]
+            )
+
+            # Search for points with the given content_id
+            search_result = self.client.scroll(
+                collection_name=self.collection,
+                scroll_filter=filter_condition,
+                limit=10000,  # Get all matching points
+                with_payload=True,
+                with_vectors=False,
+            )
+
+            if not search_result[0]:  # search_result is a tuple (points, next_page_offset)
+                log_error(f"No documents found with content_id: {content_id}")
+                return
+
+            points = search_result[0]
+            update_operations = []
+
+            # Prepare update operations for each point
+            for point in points:
+                point_id = point.id
+                current_payload = point.payload or {}
+
+                # Merge existing metadata with new metadata
+                updated_payload = current_payload.copy()
+                updated_payload.update(metadata)
+
+                if "filters" not in updated_payload:
+                    updated_payload["filters"] = {}
+                if isinstance(updated_payload["filters"], dict):
+                    updated_payload["filters"].update(metadata)
+                else:
+                    updated_payload["filters"] = metadata
+
+                # Create set payload operation
+                update_operations.append(models.SetPayload(payload=updated_payload, points=[point_id]))
+
+            # Execute all updates
+            for operation in update_operations:
+                self.client.set_payload(
+                    collection_name=self.collection, payload=operation.payload, points=operation.points
+                )
+
+            log_debug(f"Updated metadata for {len(update_operations)} documents with content_id: {content_id}")
+
+        except Exception as e:
+            log_error(f"Error updating metadata for content_id '{content_id}': {e}")
+            raise
+
     def close(self) -> None:
         """Close the Qdrant client connections."""
         if self._client is not None: