swarmauri_vectorstore_cloudweaviate 0.6.0.dev154__tar.gz

swarmauri_vectorstore_cloudweaviate-0.6.0.dev154/PKG-INFO

@@ -0,0 +1,21 @@
+Metadata-Version: 2.3
+Name: swarmauri_vectorstore_cloudweaviate
+Version: 0.6.0.dev154
+Summary: Swarmauri Weaviate Vector Store
+License: Apache-2.0
+Author: Jacob Stewart
+Author-email: jacob@swarmauri.com
+Requires-Python: >=3.10,<3.13
+Classifier: License :: OSI Approved :: Apache Software License
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Requires-Dist: swarmauri_base (>=0.6.0.dev154,<0.7.0)
+Requires-Dist: swarmauri_core (>=0.6.0.dev154,<0.7.0)
+Requires-Dist: swarmauri_embedding_doc2vec (>=0.6.0.dev154,<0.7.0)
+Requires-Dist: weaviate-client (>=4.9.2,<5.0.0)
+Project-URL: Repository, http://github.com/swarmauri/swarmauri-sdk
+Description-Content-Type: text/markdown
+
+# Swarmauri Example Community Package

swarmauri_vectorstore_cloudweaviate-0.6.0.dev154/README.md

@@ -0,0 +1 @@
+# Swarmauri Example Community Package

swarmauri_vectorstore_cloudweaviate-0.6.0.dev154/pyproject.toml

@@ -0,0 +1,58 @@
+[tool.poetry]
+name = "swarmauri_vectorstore_cloudweaviate"
+version = "0.6.0.dev154"
+description = "Swarmauri Weaviate Vector Store"
+authors = ["Jacob Stewart <jacob@swarmauri.com>"]
+license = "Apache-2.0"
+readme = "README.md"
+repository = "http://github.com/swarmauri/swarmauri-sdk"
+classifiers = [
+    "License :: OSI Approved :: Apache Software License",
+    "Programming Language :: Python :: 3.10",
+    "Programming Language :: Python :: 3.11",
+    "Programming Language :: Python :: 3.12"
+]
+
+[tool.poetry.dependencies]
+python = ">=3.10,<3.13"
+
+# Swarmauri
+swarmauri_core = {version = "^0.6.0.dev154"}
+swarmauri_base = {version = "^0.6.0.dev154"}
+swarmauri_embedding_doc2vec = {version = "^0.6.0.dev154"}
+
+# Dependencies
+weaviate-client = "^4.9.2"
+
+
+[tool.poetry.group.dev.dependencies]
+flake8 = "^7.0"
+pytest = "^8.0"
+pytest-asyncio = ">=0.24.0"
+pytest-xdist = "^3.6.1"
+pytest-json-report = "^1.5.0"
+python-dotenv = "*"
+requests = "^2.32.3"
+
+[build-system]
+requires = ["poetry-core>=1.0.0"]
+build-backend = "poetry.core.masonry.api"
+
+[tool.pytest.ini_options]
+norecursedirs = ["combined", "scripts"]
+
+markers = [
+    "test: standard test",
+    "unit: Unit tests",
+    "integration: Integration tests",
+    "acceptance: Acceptance tests",
+    "experimental: Experimental tests"
+]
+log_cli = true
+log_cli_level = "INFO"
+log_cli_format = "%(asctime)s [%(levelname)s] %(message)s"
+log_cli_date_format = "%Y-%m-%d %H:%M:%S"
+asyncio_default_fixture_loop_scope = "function"
+
+[tool.poetry.plugins."swarmauri.vector_stores"]
+CloudWeaviateVectorStore = "swarmauri_vectorstore_cloudweaviate.CloudWeaviateVectorStore:CloudWeaviateVectorStore"

swarmauri_vectorstore_cloudweaviate-0.6.0.dev154/swarmauri_vectorstore_cloudweaviate/CloudWeaviateVectorStore.py

@@ -0,0 +1,232 @@
+from typing import List, Union, Literal, Optional
+from swarmauri_core.ComponentBase import ComponentBase
+from pydantic import PrivateAttr
+import uuid as ud
+import weaviate
+from weaviate.classes.init import Auth
+from weaviate.util import generate_uuid5
+from weaviate.classes.query import MetadataQuery
+
+from swarmauri_standard.documents.Document import Document
+from swarmauri_embedding_doc2vec.Doc2VecEmbedding import Doc2VecEmbedding
+from swarmauri_standard.vectors.Vector import Vector
+
+from swarmauri_base.vector_stores.VectorStoreBase import VectorStoreBase
+from swarmauri_base.vector_stores.VectorStoreRetrieveMixin import (
+    VectorStoreRetrieveMixin,
+)
+from swarmauri_base.vector_stores.VectorStoreSaveLoadMixin import (
+    VectorStoreSaveLoadMixin,
+)
+from swarmauri_base.vector_stores.VectorStoreCloudMixin import VectorStoreCloudMixin
+
+
+@ComponentBase.register_type(VectorStoreBase, "CloudWeaviateVectorStore")
+class CloudWeaviateVectorStore(
+    VectorStoreSaveLoadMixin,
+    VectorStoreRetrieveMixin,
+    VectorStoreBase,
+    VectorStoreCloudMixin,
+):
+    type: Literal["CloudWeaviateVectorStore"] = "CloudWeaviateVectorStore"
+
+    # Private attributes
+    _client: Optional[weaviate.Client] = PrivateAttr(default=None)
+    _embedder: Doc2VecEmbedding = PrivateAttr(default=None)
+    _namespace_uuid: ud.UUID = PrivateAttr(default_factory=ud.uuid4)
+
+    def __init__(self, **data):
+        super().__init__(**data)
+
+        # Initialize the vectorizer and Weaviate client
+        self._embedder = Doc2VecEmbedding(vector_size=self.vector_size)
+        # self._initialize_client()
+
+    def connect(self, **kwargs):
+        """
+        Initialize the Weaviate client.
+        """
+        if self._client is None:
+            self._client = weaviate.connect_to_weaviate_cloud(
+                cluster_url=self.url,
+                auth_credentials=Auth.api_key(self.api_key),
+                headers=kwargs.get("headers", {}),
+            )
+
+    def disconnect(self) -> None:
+        """
+        Disconnects from the Qdrant cloud vector store.
+        """
+        if self.client is not None:
+            self.client = None
+
+    def add_document(self, document: Document) -> None:
+        """
+        Add a single document to the vector store.
+
+        :param document: Document to add
+        """
+        try:
+            collection = self._client.collections.get(self.collection_name)
+
+            # Generate or use existing embedding
+            embedding = (
+                document.embedding
+                or self._embedder.fit_transform([document.content])[0]
+            )
+
+            data_object = {
+                "content": document.content,
+                "metadata": document.metadata,
+            }
+
+            # Generate UUID for document
+            uuid = (
+                str(ud.uuid5(self._namespace_uuid, document.id))
+                if document.id
+                else generate_uuid5(data_object)
+            )
+
+            collection.data.insert(
+                properties=data_object,
+                vector=embedding.value,
+                uuid=uuid,
+            )
+
+            print(f"Document '{document.id}' added to Weaviate.")
+        except Exception as e:
+            print(f"Error adding document '{document.id}': {e}")
+            raise
+
+    def add_documents(self, documents: List[Document]) -> None:
+        """
+        Add multiple documents to the vector store.
+
+        :param documents: List of documents to add
+        """
+        try:
+            for document in documents:
+                self.add_document(document)
+
+            print(f"{len(documents)} documents added to Weaviate.")
+        except Exception as e:
+            print(f"Error adding documents: {e}")
+            raise
+
+    def get_document(self, id: str) -> Union[Document, None]:
+        """
+        Retrieve a document by its ID.
+
+        :param id: Document ID
+        :return: Document object or None if not found
+        """
+        try:
+            collection = self._client.collections.get(self.collection_name)
+
+            result = collection.query.fetch_object_by_id(
+                ud.uuid5(self._namespace_uuid, id)
+            )
+
+            if result:
+                return Document(
+                    id=id,
+                    content=result.properties["content"],
+                    metadata=result.properties["metadata"],
+                )
+            return None
+        except Exception as e:
+            print(f"Error retrieving document '{id}': {e}")
+            return None
+
+    def get_all_documents(self) -> List[Document]:
+        """
+        Retrieve all documents from the vector store.
+
+        :return: List of Document objects
+        """
+        try:
+            collection = self._client.collections.get(self.collection_name)
+            # return collection
+            documents = [
+                Document(
+                    content=item.properties["content"],
+                    metadata=item.properties["metadata"],
+                    embedding=Vector(value=list(item.vector.values())[0]),
+                )
+                for item in collection.iterator(include_vector=True)
+            ]
+            return documents
+        except Exception as e:
+            print(f"Error retrieving all documents: {e}")
+            return []
+
+    def delete_document(self, id: str) -> None:
+        """
+        Delete a document by its ID.
+
+        :param id: Document ID
+        """
+        try:
+            collection = self._client.collections.get(self.collection_name)
+            collection.data.delete_by_id(ud.uuid5(self._namespace_uuid, id))
+            print(f"Document '{id}' has been deleted from Weaviate.")
+        except Exception as e:
+            print(f"Error deleting document '{id}': {e}")
+            raise
+
+    def update_document(self, id: str, document: Document) -> None:
+        """
+        Update an existing document.
+
+        :param id: Document ID
+        :param updated_document: Document object with updated data
+        """
+        self.delete_document(id)
+        self.add_document(document)
+
+    def retrieve(self, query: str, top_k: int = 5) -> List[Document]:
+        """
+        Retrieve the top_k most relevant documents based on the given query.
+
+        :param query: Query string
+        :param top_k: Number of top similar documents to retrieve
+        :return: List of Document objects
+        """
+        try:
+            collection = self._client.collections.get(self.collection_name)
+            query_vector = self._embedder.infer_vector(query)
+            response = collection.query.near_vector(
+                near_vector=query_vector.value,
+                limit=top_k,
+                return_metadata=MetadataQuery(distance=True),
+            )
+
+            documents = [
+                Document(
+                    # id=res.id,
+                    content=res.properties["content"],
+                    metadata=res.properties["metadata"],
+                )
+                for res in response.objects
+            ]
+            return documents
+        except Exception as e:
+            print(f"Error retrieving documents for query '{query}': {e}")
+            return []
+
+    def close(self):
+        """
+        Close the connection to the Weaviate server.
+        """
+        if self._client:
+            self._client.close()
+
+    def model_dump_json(self, *args, **kwargs) -> str:
+        # Call the disconnect method before serialization
+        self.disconnect()
+
+        # Now proceed with the usual JSON serialization
+        return super().model_dump_json(*args, **kwargs)
+
+    def __del__(self):
+        self.close()

swarmauri_vectorstore_cloudweaviate-0.6.0.dev154/swarmauri_vectorstore_cloudweaviate/__init__.py

@@ -0,0 +1,12 @@
+from .CloudWeaviateVectorStore import CloudWeaviateVectorStore
+
+__version__ = "0.6.0.dev26"
+__long_desc__ = """
+
+# Swarmauri CloudWeaviate Vector Store Plugin
+
+Visit us at: https://swarmauri.com
+Follow us at: https://github.com/swarmauri
+Star us at: https://github.com/swarmauri/swarmauri-sdk
+
+"""