agno 1.8.1__py3-none-any.whl → 2.0.0rc1__py3-none-any.whl

agno/vectordb/clickhouse/clickhousedb.py

@@ -1,3 +1,4 @@
+import asyncio
 from hashlib import md5
 from typing import Any, Dict, List, Optional
 
@@ -10,8 +11,8 @@ try:
 except ImportError:
     raise ImportError("`clickhouse-connect` not installed. Use `pip install clickhouse-connect` to install it")
 
-from agno.document import Document
-from agno.embedder import Embedder
+from agno.knowledge.document import Document
+from agno.knowledge.embedder import Embedder
 from agno.utils.log import log_debug, log_info, logger
 from agno.vectordb.base import VectorDb
 from agno.vectordb.distance import Distance
@@ -62,7 +63,7 @@ class Clickhouse(VectorDb):
         # Embedder for embedding the document contents
         _embedder = embedder
         if _embedder is None:
-            from agno.embedder.openai import OpenAIEmbedder
+            from agno.knowledge.embedder.openai import OpenAIEmbedder
 
             _embedder = OpenAIEmbedder()
             log_info("Embedder not provided, using OpenAIEmbedder as default.")
@@ -157,6 +158,7 @@ class Clickhouse(VectorDb):
                     meta_data JSON DEFAULT '{{}}',
                     filters JSON DEFAULT '{{}}',
                     content String,
+                    content_id String,
                     embedding Array(Float32),
                     usage JSON,
                     created_at DateTime('UTC') DEFAULT now(),
@@ -201,6 +203,7 @@ class Clickhouse(VectorDb):
                     meta_data JSON DEFAULT '{{}}',
                     filters JSON DEFAULT '{{}}',
                     content String,
+                    content_id String,
                     embedding Array(Float32),
                     usage JSON,
                     created_at DateTime('UTC') DEFAULT now(),
@@ -211,37 +214,6 @@ class Clickhouse(VectorDb):
                 parameters=parameters,
             )
 
-    def doc_exists(self, document: Document) -> bool:
-        """
-        Validating if the document exists or not
-
-        Args:
-            document (Document): Document to validate
-        """
-        cleaned_content = document.content.replace("\x00", "\ufffd")
-        parameters = self._get_base_parameters()
-        parameters["content_hash"] = md5(cleaned_content.encode()).hexdigest()
-
-        result = self.client.query(
-            "SELECT content_hash FROM {database_name:Identifier}.{table_name:Identifier} WHERE content_hash = {content_hash:String}",
-            parameters=parameters,
-        )
-        return bool(result.result_rows)
-
-    async def async_doc_exists(self, document: Document) -> bool:
-        """Check if a document exists asynchronously."""
-        cleaned_content = document.content.replace("\x00", "\ufffd")
-        async_client = await self._ensure_async_client()
-
-        parameters = self._get_base_parameters()
-        parameters["content_hash"] = md5(cleaned_content.encode()).hexdigest()
-
-        result = await async_client.query(
-            "SELECT content_hash FROM {database_name:Identifier}.{table_name:Identifier} WHERE content_hash = {content_hash:String}",
-            parameters=parameters,
-        )
-        return bool(result.result_rows)
-
     def name_exists(self, name: str) -> bool:
         """
         Validate if a row with this name exists or not
@@ -289,6 +261,7 @@ class Clickhouse(VectorDb):
 
     def insert(
         self,
+        content_hash: str,
         documents: List[Document],
         filters: Optional[Dict[str, Any]] = None,
     ) -> None:
@@ -296,8 +269,7 @@ class Clickhouse(VectorDb):
         for document in documents:
             document.embed(embedder=self.embedder)
             cleaned_content = document.content.replace("\x00", "\ufffd")
-            content_hash = md5(cleaned_content.encode()).hexdigest()
-            _id = document.id or content_hash
+            _id = md5(cleaned_content.encode()).hexdigest()
 
             row: List[Any] = [
                 _id,
@@ -305,6 +277,7 @@ class Clickhouse(VectorDb):
                 document.meta_data,
                 filters,
                 cleaned_content,
+                document.content_id,
                 document.embedding,
                 document.usage,
                 content_hash,
@@ -320,6 +293,7 @@ class Clickhouse(VectorDb):
                 "meta_data",
                 "filters",
                 "content",
+                "content_id",
                 "embedding",
                 "usage",
                 "content_hash",
@@ -327,16 +301,19 @@ class Clickhouse(VectorDb):
         )
         log_debug(f"Inserted {len(documents)} documents")
 
-    async def async_insert(self, documents: List[Document], filters: Optional[Dict[str, Any]] = None) -> None:
+    async def async_insert(
+        self, content_hash: str, documents: List[Document], filters: Optional[Dict[str, Any]] = None
+    ) -> None:
         """Insert documents asynchronously."""
         rows: List[List[Any]] = []
         async_client = await self._ensure_async_client()
 
+        embed_tasks = [document.async_embed(embedder=self.embedder) for document in documents]
+        await asyncio.gather(*embed_tasks, return_exceptions=True)
+
         for document in documents:
-            document.embed(embedder=self.embedder)
             cleaned_content = document.content.replace("\x00", "\ufffd")
-            content_hash = md5(cleaned_content.encode()).hexdigest()
-            _id = document.id or content_hash
+            _id = md5(cleaned_content.encode()).hexdigest()
 
             row: List[Any] = [
                 _id,
@@ -344,6 +321,7 @@ class Clickhouse(VectorDb):
                 document.meta_data,
                 filters,
                 cleaned_content,
+                document.content_id,
                 document.embedding,
                 document.usage,
                 content_hash,
@@ -359,6 +337,7 @@ class Clickhouse(VectorDb):
                 "meta_data",
                 "filters",
                 "content",
+                "content_id",
                 "embedding",
                 "usage",
                 "content_hash",
@@ -371,6 +350,20 @@ class Clickhouse(VectorDb):
 
     def upsert(
         self,
+        content_hash: str,
+        documents: List[Document],
+        filters: Optional[Dict[str, Any]] = None,
+    ) -> None:
+        """
+        Upsert documents into the database.
+        """
+        if self.content_hash_exists(content_hash):
+            self._delete_by_content_hash(content_hash)
+        self.insert(content_hash=content_hash, documents=documents, filters=filters)
+
+    def _upsert(
+        self,
+        content_hash: str,
         documents: List[Document],
         filters: Optional[Dict[str, Any]] = None,
     ) -> None:
@@ -384,7 +377,7 @@ class Clickhouse(VectorDb):
         """
         # We are using ReplacingMergeTree engine in our table, so we need to insert the documents,
         # then call SELECT with FINAL
-        self.insert(documents=documents, filters=filters)
+        self.insert(content_hash=content_hash, documents=documents, filters=filters)
 
         parameters = self._get_base_parameters()
         self.client.query(
@@ -392,11 +385,21 @@ class Clickhouse(VectorDb):
             parameters=parameters,
         )
 
-    async def async_upsert(self, documents: List[Document], filters: Optional[Dict[str, Any]] = None) -> None:
+    async def async_upsert(
+        self, content_hash: str, documents: List[Document], filters: Optional[Dict[str, Any]] = None
+    ) -> None:
+        """Upsert documents asynchronously."""
+        if self.content_hash_exists(content_hash):
+            self._delete_by_content_hash(content_hash)
+        await self._async_upsert(content_hash=content_hash, documents=documents, filters=filters)
+
+    async def _async_upsert(
+        self, content_hash: str, documents: List[Document], filters: Optional[Dict[str, Any]] = None
+    ) -> None:
         """Upsert documents asynchronously."""
         # We are using ReplacingMergeTree engine in our table, so we need to insert the documents,
         # then call SELECT with FINAL
-        await self.async_insert(documents=documents, filters=filters)
+        await self.async_insert(content_hash=content_hash, documents=documents, filters=filters)
 
         parameters = self._get_base_parameters()
         await self.async_client.query(  # type: ignore
@@ -412,13 +415,6 @@ class Clickhouse(VectorDb):
 
         parameters = self._get_base_parameters()
         where_query = ""
-        # if filters:
-        #     query_filters: List[str] = []
-        #     for key, value in filters.values():
-        #         query_filters.append(f"{{{key}_key:String}} = {{{key}_value:String}}")
-        #         parameters[f"{key}_key"] = key
-        #         parameters[f"{key}_value"] = value
-        #     where_query = f"WHERE {' AND '.join(query_filters)}"
 
         order_by_query = ""
         if self.distance == Distance.l2 or self.distance == Distance.max_inner_product:
@@ -429,7 +425,7 @@ class Clickhouse(VectorDb):
             parameters["query_embedding"] = query_embedding
 
         clickhouse_query = (
-            "SELECT name, meta_data, content, embedding, usage FROM "
+            "SELECT name, meta_data, content, content_id, embedding, usage FROM "
             "{database_name:Identifier}.{table_name:Identifier} "
             f"{where_query} {order_by_query} LIMIT {limit}"
         )
@@ -455,9 +451,10 @@ class Clickhouse(VectorDb):
                     name=result[0],
                     meta_data=result[1],
                     content=result[2],
+                    content_id=result[3],
                     embedder=self.embedder,
-                    embedding=result[3],
-                    usage=result[4],
+                    embedding=result[4],
+                    usage=result[5],
                 )
             )
 
@@ -476,13 +473,6 @@ class Clickhouse(VectorDb):
 
         parameters = self._get_base_parameters()
         where_query = ""
-        # if filters:
-        #     query_filters: List[str] = []
-        #     for key, value in filters.values():
-        #         query_filters.append(f"{{{key}_key:String}} = {{{key}_value:String}}")
-        #         parameters[f"{key}_key"] = key
-        #         parameters[f"{key}_value"] = value
-        #     where_query = f"WHERE {' AND '.join(query_filters)}"
 
         order_by_query = ""
         if self.distance == Distance.l2 or self.distance == Distance.max_inner_product:
@@ -493,7 +483,7 @@ class Clickhouse(VectorDb):
             parameters["query_embedding"] = query_embedding
 
         clickhouse_query = (
-            "SELECT name, meta_data, content, embedding, usage FROM "
+            "SELECT name, meta_data, content, content_id, embedding, usage FROM "
             "{database_name:Identifier}.{table_name:Identifier} "
             f"{where_query} {order_by_query} LIMIT {limit}"
         )
@@ -519,9 +509,10 @@ class Clickhouse(VectorDb):
                     name=result[0],
                     meta_data=result[1],
                     content=result[2],
+                    content_id=result[3],
                     embedder=self.embedder,
-                    embedding=result[3],
-                    usage=result[4],
+                    embedding=result[4],
+                    usage=result[5],
                 )
             )
 
@@ -573,3 +564,219 @@ class Clickhouse(VectorDb):
             parameters=parameters,
         )
         return True
+
+    def delete_by_id(self, id: str) -> bool:
+        """
+
+        Delete a document by its ID.
+
+        Args:
+            id (str): The document ID to delete
+
+        Returns:
+            bool: True if document was deleted, False otherwise
+        """
+        try:
+            log_debug(f"ClickHouse VectorDB : Deleting document with ID {id}")
+            if not self.id_exists(id):
+                return False
+
+            parameters = self._get_base_parameters()
+            parameters["id"] = id
+
+            self.client.command(
+                "DELETE FROM {database_name:Identifier}.{table_name:Identifier} WHERE id = {id:String}",
+                parameters=parameters,
+            )
+            return True
+        except Exception as e:
+            log_info(f"Error deleting document with ID {id}: {e}")
+            return False
+
+    def delete_by_name(self, name: str) -> bool:
+        """
+        Delete documents by name.
+
+        Args:
+            name (str): The document name to delete
+
+        Returns:
+            bool: True if documents were deleted, False otherwise
+        """
+        try:
+            log_debug(f"ClickHouse VectorDB : Deleting documents with name {name}")
+            if not self.name_exists(name):
+                return False
+
+            parameters = self._get_base_parameters()
+            parameters["name"] = name
+
+            self.client.command(
+                "DELETE FROM {database_name:Identifier}.{table_name:Identifier} WHERE name = {name:String}",
+                parameters=parameters,
+            )
+            return True
+        except Exception as e:
+            log_info(f"Error deleting documents with name {name}: {e}")
+            return False
+
+    def delete_by_metadata(self, metadata: Dict[str, Any]) -> bool:
+        """
+        Delete documents by metadata.
+
+        Args:
+            metadata (Dict[str, Any]): The metadata to match for deletion
+
+        Returns:
+            bool: True if documents were deleted, False otherwise
+        """
+        try:
+            log_debug(f"ClickHouse VectorDB : Deleting documents with metadata {metadata}")
+            parameters = self._get_base_parameters()
+
+            # Build WHERE clause for metadata matching using proper ClickHouse JSON syntax
+            where_conditions = []
+            for key, value in metadata.items():
+                if isinstance(value, bool):
+                    where_conditions.append(f"JSONExtractBool(toString(filters), '{key}') = {str(value).lower()}")
+                elif isinstance(value, (int, float)):
+                    where_conditions.append(f"JSONExtractFloat(toString(filters), '{key}') = {value}")
+                else:
+                    where_conditions.append(f"JSONExtractString(toString(filters), '{key}') = '{value}'")
+
+            if not where_conditions:
+                return False
+
+            where_clause = " AND ".join(where_conditions)
+
+            self.client.command(
+                f"DELETE FROM {{database_name:Identifier}}.{{table_name:Identifier}} WHERE {where_clause}",
+                parameters=parameters,
+            )
+            return True
+        except Exception as e:
+            log_info(f"Error deleting documents with metadata {metadata}: {e}")
+            return False
+
+    def delete_by_content_id(self, content_id: str) -> bool:
+        """
+        Delete documents by content ID.
+
+        Args:
+            content_id (str): The content ID to delete
+
+        Returns:
+            bool: True if documents were deleted, False otherwise
+        """
+        try:
+            log_debug(f"ClickHouse VectorDB : Deleting documents with content_id {content_id}")
+            parameters = self._get_base_parameters()
+            parameters["content_id"] = content_id
+
+            self.client.command(
+                "DELETE FROM {database_name:Identifier}.{table_name:Identifier} WHERE content_id = {content_id:String}",
+                parameters=parameters,
+            )
+            return True
+        except Exception as e:
+            log_info(f"Error deleting documents with content_id {content_id}: {e}")
+            return False
+
+    def content_hash_exists(self, content_hash: str) -> bool:
+        """
+        Validate if a row with this content_hash exists or not
+
+        Args:
+            content_hash (str): Content hash to check
+        """
+        parameters = self._get_base_parameters()
+        parameters["content_hash"] = content_hash
+
+        result = self.client.query(
+            "SELECT content_hash FROM {database_name:Identifier}.{table_name:Identifier} WHERE content_hash = {content_hash:String}",
+            parameters=parameters,
+        )
+        return bool(result)
+
+    def _delete_by_content_hash(self, content_hash: str) -> bool:
+        """
+        Delete documents by content hash.
+        """
+        try:
+            parameters = self._get_base_parameters()
+            parameters["content_hash"] = content_hash
+
+            self.client.command(
+                "DELETE FROM {database_name:Identifier}.{table_name:Identifier} WHERE content_hash = {content_hash:String}",
+                parameters=parameters,
+            )
+            return True
+        except Exception:
+            return False
+
+    def update_metadata(self, content_id: str, metadata: Dict[str, Any]) -> None:
+        """
+        Update the metadata for documents with the given content_id.
+
+        Args:
+            content_id (str): The content ID to update
+            metadata (Dict[str, Any]): The metadata to update
+        """
+        import json
+
+        try:
+            parameters = self._get_base_parameters()
+            parameters["content_id"] = content_id
+
+            # First, get existing documents with their current metadata and filters
+            result = self.client.query(
+                "SELECT id, meta_data, filters FROM {database_name:Identifier}.{table_name:Identifier} WHERE content_id = {content_id:String}",
+                parameters=parameters,
+            )
+
+            if not result.result_rows:
+                logger.debug(f"No documents found with content_id: {content_id}")
+                return
+
+            # Update each document
+            updated_count = 0
+            for row in result.result_rows:
+                doc_id, current_meta_json, current_filters_json = row
+
+                # Parse existing metadata
+                try:
+                    current_metadata = json.loads(current_meta_json) if current_meta_json else {}
+                except (json.JSONDecodeError, TypeError):
+                    current_metadata = {}
+
+                # Parse existing filters
+                try:
+                    current_filters = json.loads(current_filters_json) if current_filters_json else {}
+                except (json.JSONDecodeError, TypeError):
+                    current_filters = {}
+
+                # Merge existing metadata with new metadata
+                updated_metadata = current_metadata.copy()
+                updated_metadata.update(metadata)
+
+                # Merge existing filters with new metadata
+                updated_filters = current_filters.copy()
+                updated_filters.update(metadata)
+
+                # Update the document
+                update_params = parameters.copy()
+                update_params["doc_id"] = doc_id
+                update_params["metadata_json"] = json.dumps(updated_metadata)
+                update_params["filters_json"] = json.dumps(updated_filters)
+
+                self.client.command(
+                    "ALTER TABLE {database_name:Identifier}.{table_name:Identifier} UPDATE meta_data = {metadata_json:String}, filters = {filters_json:String} WHERE id = {doc_id:String}",
+                    parameters=update_params,
+                )
+                updated_count += 1
+
+            logger.debug(f"Updated metadata for {updated_count} documents with content_id: {content_id}")
+
+        except Exception as e:
+            logger.error(f"Error updating metadata for content_id '{content_id}': {e}")
+            raise