swarmauri_vectorstore_redis 0.6.0.dev154__py3-none-any.whl

swarmauri_vectorstore_redis/RedisDocumentRetriever.py

@@ -0,0 +1,59 @@
+from typing import List
+from redisearch import Client, Query
+from swarmauri_core.documents.IDocument import IDocument
+from swarmauri_standard.document_stores.ConcreteDocument import (
+    ConcreteDocument,
+)
+from swarmauri_base.retrievers.DocumentRetrieverBase import DocumentRetrieverBase
+from swarmauri_core.ComponentBase import ComponentBase
+
+@ComponentBase.register_type(DocumentRetrieverBase, "RedisDocumentRetriever")
+class RedisDocumentRetriever(DocumentRetrieverBase):
+    """
+    A document retriever that fetches documents from a Redis store.
+    """
+
+    def __init__(self, redis_idx_name, redis_host, redis_port):
+        """
+        Initializes a new instance of RedisDocumentRetriever.
+
+        Args:
+            redis_client (Redis): An instance of the Redis client.
+        """
+        self._redis_client = None
+        self._redis_idx_name = redis_idx_name
+        self._redis_host = redis_host
+        self._redis_port = redis_port
+
+    @property
+    def redis_client(self):
+        """Lazily initialize and return the Redis client using a factory method."""
+        if self._redis_client is None:
+            self._redis_client = Client(
+                self.redis_idx_name, host=self.redis_host, port=self.redis_port
+            )
+        return self._redis_client
+
+    def retrieve(self, query: str, top_k: int = 5) -> List[IDocument]:
+        """
+        Retrieve the most relevant documents based on the given query.
+
+        Args:
+            query (str): The query string used for document retrieval.
+            top_k (int, optional): The number of top relevant documents to retrieve. Defaults to 5.
+
+        Returns:
+            List[IDocument]: A list of the top_k most relevant documents.
+        """
+        query_result = self.redis_client.search(Query(query).paging(0, top_k))
+
+        documents = [
+            ConcreteDocument(
+                doc_id=doc.id,
+                content=doc.text,  # Note: Adjust 'text' based on actual Redis document schema
+                metadata=doc.__dict__,  # Including full document fields and values in metadata
+            )
+            for doc in query_result.docs
+        ]
+
+        return documents

swarmauri_vectorstore_redis/RedisVectorStore.py

@@ -0,0 +1,238 @@
+import json
+from typing import List, Union, Literal, Optional
+from pydantic import PrivateAttr
+
+import numpy as np
+import redis
+from redis.commands.search.field import VectorField, TextField
+from redis.commands.search.indexDefinition import IndexDefinition, IndexType
+
+from swarmauri_standard.vectors.Vector import Vector
+from swarmauri_standard.documents.concrete.Document import Document
+from swarmauri_embedding_doc2vec.Doc2VecEmbedding import Doc2VecEmbedding
+from swarmauri_base.vector_stores.VectorStoreBase import VectorStoreBase
+from swarmauri_base.vector_stores.VectorStoreRetrieveMixin import VectorStoreRetrieveMixin
+from swarmauri_base.vector_stores.VectorStoreSaveLoadMixin import VectorStoreSaveLoadMixin
+from swarmauri_core.ComponentBase import ComponentBase
+
+@ComponentBase.register_type(VectorStoreBase, "RedisVectorStore")
+class RedisVectorStore(VectorStoreSaveLoadMixin, VectorStoreRetrieveMixin, VectorStoreBase):
+    type: Literal["RedisVectorStore"] = "RedisVectorStore"
+    index_name: str = "documents_index"
+    embedding_dimension: int = 8000  # Default embedding dimension
+
+    # Private attributes
+    _embedder: Doc2VecEmbedding = PrivateAttr()
+    _redis_client: Optional[redis.Redis] = PrivateAttr(default=None)
+
+    # Configuration attributes with default values
+    redis_host: str = "localhost"
+    redis_port: int = 6379
+    redis_password: Optional[str] = None
+
+    def __init__(self, **kwargs):
+        super().__init__(**kwargs)
+        self._embedder = Doc2VecEmbedding(vector_size=self.embedding_dimension)
+
+        # Initialize Redis client using class attributes
+        self.connect()
+
+        # Setup Redis Search index
+        vector_field = VectorField(
+            "embedding",
+            "FLAT",
+            {
+                "TYPE": "FLOAT32",
+                "DIM": self.embedding_dimension,
+                "DISTANCE_METRIC": "COSINE"
+            }
+        )
+        text_field = TextField("content")
+
+        try:
+            self._redis_client.ft(self.index_name).info()
+            print(f"Index '{self.index_name}' exists.")
+        except Exception:
+            print(f"Index '{self.index_name}' does not exist. Creating index...")
+            schema = (
+                text_field,
+                vector_field
+            )
+            definition = IndexDefinition(
+                prefix=["doc:"],
+                index_type=IndexType.HASH
+            )
+            self._redis_client.ft(self.index_name).create_index(
+                fields=schema,
+                definition=definition
+            )
+            print(f"Index '{self.index_name}' created successfully.")
+
+
+    def connect(self) -> None:
+        """
+        Establishes a connection to the Redis server using class attributes.
+        """
+        try:
+            self._redis_client = redis.Redis(
+                host=self.redis_host,
+                port=self.redis_port,
+                password=self.redis_password,
+                decode_responses=False,  # For binary data
+            )
+            # Test the connection
+            self._redis_client.ping()
+            print("Connected to Redis successfully.")
+        except Exception as e:
+            print(f"Failed to connect to Redis: {e}")
+            raise
+
+    def disconnect(self) -> None:
+        """
+        Disconnects from the Redis server.
+        """
+        if self._redis_client:
+            self._redis_client.close()
+            self._redis_client = None
+            print("Disconnected from Redis.")
+       
+
+    def _doc_key(self, document_id: str) -> str:
+        return f"doc:{document_id}"
+
+    def add_document(self, document: Document) -> None:
+        doc = document
+        pipeline = self._redis_client.pipeline()
+    
+            # Embed the document content
+        embedding = self._embedder.fit_transform([doc.content])[0]
+
+        if isinstance(embedding, Vector):
+            embedding = embedding.value 
+        metadata = doc.metadata
+               
+        # print("METADATA ::::::::::::::::::::", metadata)
+        doc_key = self._doc_key(doc.id)
+        # print("DOC KEY ::::::::::::::::::::", doc_key)
+        pipeline.hset(doc_key, mapping={
+            "content": doc.content,
+            "metadata": json.dumps(metadata),  # Store metadata as JSON
+            "embedding": np.array(embedding, dtype=np.float32).tobytes()  # Convert embedding values to bytes
+        })
+        add = pipeline.execute()
+
+    def add_documents(self, documents: List[Document]) -> None:
+        pipeline = self._redis_client.pipeline()
+        for doc in documents:
+            if not doc.content:
+                continue
+            # Embed the document content
+            embedding = self._embedder.fit_transform([doc.content])[0]
+            
+            if isinstance(embedding, Vector):
+                embedding = embedding.value
+            metadata={doc.metadata}
+           
+            doc_key = self._doc_key(doc.id)
+            pipeline.hset(doc_key, mapping={
+                "content": doc.content,
+                "metadata": json.dumps(metadata),
+                "embedding": np.array(embedding, dtype=np.float32).tobytes()
+            })
+        pipeline.execute()
+
+    def get_document(self, id: str) -> Union[Document, None]:
+        
+        doc_key = self._doc_key(id)
+        data = self._redis_client.hgetall(doc_key)
+        if not data:
+            return None
+        
+        metadata_raw = data.get(b"metadata", b"{}").decode("utf-8")  
+        metadata = json.loads(metadata_raw)
+
+        content = data.get(b"content", b"").decode("utf-8")
+        # print("METAAAAAAA ::::::::::::", metadata)
+
+        embedding_bytes = data.get(b"embedding")
+        if embedding_bytes:
+            embedding = Vector(value=np.frombuffer(embedding_bytes, dtype=np.float32).tolist())
+        else:
+            embedding = None
+        return Document(
+            id=id,
+            content=content,
+            metadata=metadata,
+            embedding=embedding
+        )
+
+    def get_all_documents(self) -> List[Document]:
+        cursor = '0'
+        documents = []
+        while cursor != 0:
+            cursor, keys = self._redis_client.scan(cursor=cursor, match="doc:*", count=1000)
+            for key in keys:
+                data = self._redis_client.hgetall(key)
+                if not data:
+                    continue
+                doc_id = key.decode("utf-8").split("doc:")[1]
+                metadata_raw = data.get(b"metadata", b"{}").decode("utf-8")  
+                metadata = json.loads(metadata_raw)
+                content = data.get(b"content", b"").decode("utf-8")
+                embedding_bytes = data.get(b"embedding")
+                if embedding_bytes:
+                    embedding = Vector(value=np.frombuffer(embedding_bytes, dtype=np.float32).tolist())
+                else:
+                    embedding = None
+                document = Document(
+                    id=doc_id,
+                    content=content,
+                    metadata=metadata,
+                    embedding=embedding
+                )
+                documents.append(document)
+        return documents
+
+    def delete_document(self, id: str) -> None:
+        doc_key = self._doc_key(id)
+        self._redis_client.delete(doc_key)
+
+    def update_document(self, document: Document) -> None:
+        doc_key = self._doc_key(document.id)
+        if not self._redis_client.exists(doc_key):
+            raise ValueError(f"Document with id {document.id} does not exist.")
+        # Update the document by re-adding it
+        self.add_documents([document])
+        
+
+    def cosine_similarity(self, vec1, vec2):
+        dot_product = np.dot(vec1, vec2)
+        norm_vec1 = np.linalg.norm(vec1)
+        norm_vec2 = np.linalg.norm(vec2)
+        if norm_vec1 == 0 or norm_vec2 == 0:
+            return 0
+        return dot_product / (norm_vec1 * norm_vec2)
+
+
+    def retrieve(self, query: str, top_k: int = 5) -> List[Document]:
+        query_vector = self._embedder.infer_vector(query)
+            
+        all_documents = self.get_all_documents()
+        # print("ALL DOCUMENTS ::::::::::::::::::::", all_documents[:10])
+        similarities = []
+        for doc in all_documents:
+            if doc.embedding is not None:
+                doc_vector = doc.embedding
+                # print("DOC VECTOR ::::::::::::::::::::", doc_vector.value[:10])
+                similarity = self.cosine_similarity(query_vector.value, doc_vector.value)
+                similarities.append((doc, similarity))
+        
+        similarities.sort(key=lambda x: x[1], reverse=True)
+        # print("SIMILARITIES ::::::::::::::::::::", similarities[:10])
+        top_documents = [doc for doc, _ in similarities[:top_k]]
+        # print(f"Found {len(top_documents)} similar documents.")
+        return top_documents
+        
+    
+    class Config:
+        extra = 'allow'

swarmauri_vectorstore_redis/__init__.py

@@ -0,0 +1,12 @@
+from .RedisVectorStore import RedisVectorStore
+
+__version__ = "0.6.0.dev26"
+__long_desc__ = """
+
+# Swarmauri Redis VectorStore Plugin
+
+Visit us at: https://swarmauri.com
+Follow us at: https://github.com/swarmauri
+Star us at: https://github.com/swarmauri/swarmauri-sdk
+
+"""

swarmauri_vectorstore_redis-0.6.0.dev154.dist-info/METADATA

@@ -0,0 +1,20 @@
+Metadata-Version: 2.3
+Name: swarmauri_vectorstore_redis
+Version: 0.6.0.dev154
+Summary: Swarmauri Redis Vector Store
+License: Apache-2.0
+Author: Jacob Stewart
+Author-email: jacob@swarmauri.com
+Requires-Python: >=3.10,<3.13
+Classifier: License :: OSI Approved :: Apache Software License
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Requires-Dist: redis (>=4.0,<5.0)
+Requires-Dist: swarmauri_base (>=0.6.0.dev154,<0.7.0)
+Requires-Dist: swarmauri_core (>=0.6.0.dev154,<0.7.0)
+Project-URL: Repository, http://github.com/swarmauri/swarmauri-sdk
+Description-Content-Type: text/markdown
+
+# Swarmauri Example Community Package

swarmauri_vectorstore_redis-0.6.0.dev154.dist-info/RECORD

@@ -0,0 +1,7 @@
+swarmauri_vectorstore_redis/__init__.py,sha256=kEer1rPuqj3_CAoRTjOr9-nBY7Dz6n__w_kXSddD6u4,285
+swarmauri_vectorstore_redis/RedisDocumentRetriever.py,sha256=5x62kZ2a7H8qoHzOttNnpg3WlKoyL6FyCuR8PlEm_SU,2234
+swarmauri_vectorstore_redis/RedisVectorStore.py,sha256=cmckLjTsg1qxvqojom3VZW41tIg8bOlkWPt4l1Ufsug,9121
+swarmauri_vectorstore_redis-0.6.0.dev154.dist-info/entry_points.txt,sha256=HSN4TqGUlAgs7ovkyK0DCa0k4MoQkF2irfPAPjIw9kU,227
+swarmauri_vectorstore_redis-0.6.0.dev154.dist-info/METADATA,sha256=djVzFMQXuV4whFaQwpCTeFSMKKeqrm9qmbXPgDl0hFM,774
+swarmauri_vectorstore_redis-0.6.0.dev154.dist-info/WHEEL,sha256=IYZQI976HJqqOpQU6PHkJ8fb3tMNBFjg-Cn-pwAbaFM,88
+swarmauri_vectorstore_redis-0.6.0.dev154.dist-info/RECORD,,

swarmauri_vectorstore_redis-0.6.0.dev154.dist-info/WHEEL

@@ -0,0 +1,4 @@
+Wheel-Version: 1.0
+Generator: poetry-core 2.0.1
+Root-Is-Purelib: true
+Tag: py3-none-any

swarmauri_vectorstore_redis-0.6.0.dev154.dist-info/entry_points.txt

@@ -0,0 +1,6 @@
+[swarmauri.retrievers]
+RedisDocumentRetriever=swarmauri_vectorstore_redis.RedisDocumentRetriever:RedisDocumentRetriever
+
+[swarmauri.vector_stores]
+RedisVectorStore=swarmauri_vectorstore_redis.RedisVectorStore:RedisVectorStore
+