lfx-nightly 0.1.11.dev0__py3-none-any.whl

lfx/components/vectorstores/mongodb_atlas.py

@@ -0,0 +1,213 @@
+import tempfile
+import time
+
+import certifi
+from langchain_community.vectorstores import MongoDBAtlasVectorSearch
+from pymongo.collection import Collection
+from pymongo.operations import SearchIndexModel
+
+from lfx.base.vectorstores.model import LCVectorStoreComponent, check_cached_vector_store
+from lfx.helpers.data import docs_to_data
+from lfx.io import BoolInput, DropdownInput, HandleInput, IntInput, SecretStrInput, StrInput
+from lfx.schema.data import Data
+
+
+class MongoVectorStoreComponent(LCVectorStoreComponent):
+    display_name = "MongoDB Atlas"
+    description = "MongoDB Atlas Vector Store with search capabilities"
+    name = "MongoDBAtlasVector"
+    icon = "MongoDB"
+    INSERT_MODES = ["append", "overwrite"]
+    SIMILARITY_OPTIONS = ["cosine", "euclidean", "dotProduct"]
+    QUANTIZATION_OPTIONS = ["scalar", "binary"]
+    inputs = [
+        SecretStrInput(name="mongodb_atlas_cluster_uri", display_name="MongoDB Atlas Cluster URI", required=True),
+        BoolInput(name="enable_mtls", display_name="Enable mTLS", value=False, advanced=True, required=True),
+        SecretStrInput(
+            name="mongodb_atlas_client_cert",
+            display_name="MongoDB Atlas Combined Client Certificate",
+            required=False,
+            info="Client Certificate combined with the private key in the following format:\n "
+            "-----BEGIN PRIVATE KEY-----\n...\n -----END PRIVATE KEY-----\n-----BEGIN CERTIFICATE-----\n"
+            "...\n-----END CERTIFICATE-----\n",
+        ),
+        StrInput(name="db_name", display_name="Database Name", required=True),
+        StrInput(name="collection_name", display_name="Collection Name", required=True),
+        StrInput(
+            name="index_name",
+            display_name="Index Name",
+            required=True,
+            info="The name of Atlas Search index, it should be a Vector Search.",
+        ),
+        *LCVectorStoreComponent.inputs,
+        DropdownInput(
+            name="insert_mode",
+            display_name="Insert Mode",
+            options=INSERT_MODES,
+            value=INSERT_MODES[0],
+            info="How to insert new documents into the collection.",
+            advanced=True,
+        ),
+        HandleInput(name="embedding", display_name="Embedding", input_types=["Embeddings"]),
+        IntInput(
+            name="number_of_results",
+            display_name="Number of Results",
+            info="Number of results to return.",
+            value=4,
+            advanced=True,
+        ),
+        StrInput(
+            name="index_field",
+            display_name="Index Field",
+            advanced=True,
+            required=True,
+            info="The field to index.",
+            value="embedding",
+        ),
+        StrInput(
+            name="filter_field", display_name="Filter Field", advanced=True, info="The field to filter the index."
+        ),
+        IntInput(
+            name="number_dimensions",
+            display_name="Number of Dimensions",
+            info="Embedding Context Length.",
+            value=1536,
+            advanced=True,
+            required=True,
+        ),
+        DropdownInput(
+            name="similarity",
+            display_name="Similarity",
+            options=SIMILARITY_OPTIONS,
+            value=SIMILARITY_OPTIONS[0],
+            info="The method used to measure the similarity between vectors.",
+            advanced=True,
+        ),
+        DropdownInput(
+            name="quantization",
+            display_name="Quantization",
+            options=QUANTIZATION_OPTIONS,
+            value=None,
+            info="Quantization reduces memory costs converting 32-bit floats to smaller data types",
+            advanced=True,
+        ),
+    ]
+
+    @check_cached_vector_store
+    def build_vector_store(self) -> MongoDBAtlasVectorSearch:
+        try:
+            from pymongo import MongoClient
+        except ImportError as e:
+            msg = "Please install pymongo to use MongoDB Atlas Vector Store"
+            raise ImportError(msg) from e
+
+        # Create temporary files for the client certificate
+        if self.enable_mtls:
+            client_cert_path = None
+            try:
+                client_cert = self.mongodb_atlas_client_cert.replace(" ", "\n")
+                client_cert = client_cert.replace("-----BEGIN\nPRIVATE\nKEY-----", "-----BEGIN PRIVATE KEY-----")
+                client_cert = client_cert.replace(
+                    "-----END\nPRIVATE\nKEY-----\n-----BEGIN\nCERTIFICATE-----",
+                    "-----END PRIVATE KEY-----\n-----BEGIN CERTIFICATE-----",
+                )
+                client_cert = client_cert.replace("-----END\nCERTIFICATE-----", "-----END CERTIFICATE-----")
+                with tempfile.NamedTemporaryFile(delete=False) as client_cert_file:
+                    client_cert_file.write(client_cert.encode("utf-8"))
+                    client_cert_path = client_cert_file.name
+
+            except Exception as e:
+                msg = f"Failed to write certificate to temporary file: {e}"
+                raise ValueError(msg) from e
+
+        try:
+            mongo_client: MongoClient = (
+                MongoClient(
+                    self.mongodb_atlas_cluster_uri,
+                    tls=True,
+                    tlsCertificateKeyFile=client_cert_path,
+                    tlsCAFile=certifi.where(),
+                )
+                if self.enable_mtls
+                else MongoClient(self.mongodb_atlas_cluster_uri)
+            )
+
+            collection = mongo_client[self.db_name][self.collection_name]
+
+        except Exception as e:
+            msg = f"Failed to connect to MongoDB Atlas: {e}"
+            raise ValueError(msg) from e
+
+        # Convert DataFrame to Data if needed using parent's method
+        self.ingest_data = self._prepare_ingest_data()
+
+        documents = []
+        for _input in self.ingest_data or []:
+            if isinstance(_input, Data):
+                documents.append(_input.to_lc_document())
+            else:
+                documents.append(_input)
+
+        if documents:
+            self.__insert_mode(collection)
+
+            return MongoDBAtlasVectorSearch.from_documents(
+                documents=documents, embedding=self.embedding, collection=collection, index_name=self.index_name
+            )
+        return MongoDBAtlasVectorSearch(embedding=self.embedding, collection=collection, index_name=self.index_name)
+
+    def search_documents(self) -> list[Data]:
+        from bson.objectid import ObjectId
+
+        vector_store = self.build_vector_store()
+
+        self.verify_search_index(vector_store._collection)
+
+        if self.search_query and isinstance(self.search_query, str):
+            docs = vector_store.similarity_search(
+                query=self.search_query,
+                k=self.number_of_results,
+            )
+            for doc in docs:
+                doc.metadata = {
+                    key: str(value) if isinstance(value, ObjectId) else value for key, value in doc.metadata.items()
+                }
+
+            data = docs_to_data(docs)
+            self.status = data
+            return data
+        return []
+
+    def __insert_mode(self, collection: Collection) -> None:
+        if self.insert_mode == "overwrite":
+            collection.delete_many({})  # Delete all documents while preserving collection structure
+
+    def verify_search_index(self, collection: Collection) -> None:
+        """Verify if the search index exists, if not, create it.
+
+        Args:
+            collection (Collection): The collection to verify the search index on.
+        """
+        indexes = collection.list_search_indexes()
+
+        index_names_types = {idx["name"]: idx["type"] for idx in indexes}
+        index_names = list(index_names_types.keys())
+        index_type = index_names_types.get(self.index_name)
+        if self.index_name not in index_names and index_type != "vectorSearch":
+            collection.create_search_index(self.__create_index_definition())
+
+            time.sleep(20)  # Give some time for index to be ready
+
+    def __create_index_definition(self) -> SearchIndexModel:
+        fields = [
+            {
+                "type": "vector",
+                "path": self.index_field,
+                "numDimensions": self.number_dimensions,
+                "similarity": self.similarity,
+                "quantization": self.quantization,
+            }
+        ]
+        if self.filter_field:
+            fields.append({"type": "filter", "path": self.filter_field})
+        return SearchIndexModel(definition={"fields": fields}, name=self.index_name, type="vectorSearch")

lfx/components/vectorstores/opensearch.py

@@ -0,0 +1,243 @@
+import json
+from typing import Any
+
+from langchain_community.vectorstores import OpenSearchVectorSearch
+
+from lfx.base.vectorstores.model import LCVectorStoreComponent, check_cached_vector_store
+from lfx.base.vectorstores.vector_store_connection_decorator import vector_store_connection
+from lfx.io import (
+    BoolInput,
+    DropdownInput,
+    FloatInput,
+    HandleInput,
+    IntInput,
+    MultilineInput,
+    SecretStrInput,
+    StrInput,
+)
+from lfx.schema.data import Data
+
+
+@vector_store_connection
+class OpenSearchVectorStoreComponent(LCVectorStoreComponent):
+    """OpenSearch Vector Store with advanced, customizable search capabilities."""
+
+    display_name: str = "OpenSearch"
+    description: str = "OpenSearch Vector Store with advanced, customizable search capabilities."
+    name = "OpenSearch"
+    icon = "OpenSearch"
+
+    inputs = [
+        StrInput(
+            name="opensearch_url",
+            display_name="OpenSearch URL",
+            value="http://localhost:9200",
+            info="URL for OpenSearch cluster (e.g. https://192.168.1.1:9200).",
+        ),
+        StrInput(
+            name="index_name",
+            display_name="Index Name",
+            value="langflow",
+            info="The index name where the vectors will be stored in OpenSearch cluster.",
+        ),
+        *LCVectorStoreComponent.inputs,
+        HandleInput(name="embedding", display_name="Embedding", input_types=["Embeddings"]),
+        DropdownInput(
+            name="search_type",
+            display_name="Search Type",
+            options=["similarity", "similarity_score_threshold", "mmr"],
+            value="similarity",
+            advanced=True,
+        ),
+        IntInput(
+            name="number_of_results",
+            display_name="Number of Results",
+            info="Number of results to return.",
+            advanced=True,
+            value=4,
+        ),
+        FloatInput(
+            name="search_score_threshold",
+            display_name="Search Score Threshold",
+            info="Minimum similarity score threshold for search results.",
+            value=0.0,
+            advanced=True,
+        ),
+        StrInput(
+            name="username",
+            display_name="Username",
+            value="admin",
+            advanced=True,
+        ),
+        SecretStrInput(
+            name="password",
+            display_name="Password",
+            value="admin",
+            advanced=True,
+        ),
+        BoolInput(
+            name="use_ssl",
+            display_name="Use SSL",
+            value=True,
+            advanced=True,
+        ),
+        BoolInput(
+            name="verify_certs",
+            display_name="Verify Certificates",
+            value=False,
+            advanced=True,
+        ),
+        MultilineInput(
+            name="hybrid_search_query",
+            display_name="Hybrid Search Query",
+            value="",
+            advanced=True,
+            info=(
+                "Provide a custom hybrid search query in JSON format. This allows you to combine "
+                "vector similarity and keyword matching."
+            ),
+        ),
+    ]
+
+    @check_cached_vector_store
+    def build_vector_store(self) -> OpenSearchVectorSearch:
+        """Builds the OpenSearch Vector Store object."""
+        try:
+            from langchain_community.vectorstores import OpenSearchVectorSearch
+        except ImportError as e:
+            error_message = f"Failed to import required modules: {e}"
+            self.log(error_message)
+            raise ImportError(error_message) from e
+
+        try:
+            opensearch = OpenSearchVectorSearch(
+                index_name=self.index_name,
+                embedding_function=self.embedding,
+                opensearch_url=self.opensearch_url,
+                http_auth=(self.username, self.password),
+                use_ssl=self.use_ssl,
+                verify_certs=self.verify_certs,
+                ssl_assert_hostname=False,
+                ssl_show_warn=False,
+            )
+        except Exception as e:
+            error_message = f"Failed to create OpenSearchVectorSearch instance: {e}"
+            self.log(error_message)
+            raise RuntimeError(error_message) from e
+
+        if self.ingest_data:
+            self._add_documents_to_vector_store(opensearch)
+
+        return opensearch
+
+    def _add_documents_to_vector_store(self, vector_store: "OpenSearchVectorSearch") -> None:
+        """Adds documents to the Vector Store."""
+        # Convert DataFrame to Data if needed using parent's method
+        self.ingest_data = self._prepare_ingest_data()
+
+        documents = []
+        for _input in self.ingest_data or []:
+            if isinstance(_input, Data):
+                documents.append(_input.to_lc_document())
+            else:
+                error_message = f"Expected Data object, got {type(_input)}"
+                self.log(error_message)
+                raise TypeError(error_message)
+
+        if documents and self.embedding is not None:
+            self.log(f"Adding {len(documents)} documents to the Vector Store.")
+            try:
+                vector_store.add_documents(documents)
+            except Exception as e:
+                error_message = f"Error adding documents to Vector Store: {e}"
+                self.log(error_message)
+                raise RuntimeError(error_message) from e
+        else:
+            self.log("No documents to add to the Vector Store.")
+
+    def search(self, query: str | None = None) -> list[dict[str, Any]]:
+        """Search for similar documents in the vector store or retrieve all documents if no query is provided."""
+        try:
+            vector_store = self.build_vector_store()
+
+            query = query or ""
+
+            if self.hybrid_search_query.strip():
+                try:
+                    hybrid_query = json.loads(self.hybrid_search_query)
+                except json.JSONDecodeError as e:
+                    error_message = f"Invalid hybrid search query JSON: {e}"
+                    self.log(error_message)
+                    raise ValueError(error_message) from e
+
+                results = vector_store.client.search(index=self.index_name, body=hybrid_query)
+
+                processed_results = []
+                for hit in results.get("hits", {}).get("hits", []):
+                    source = hit.get("_source", {})
+                    text = source.get("text", "")
+                    metadata = source.get("metadata", {})
+
+                    if isinstance(text, dict):
+                        text = text.get("text", "")
+
+                    processed_results.append(
+                        {
+                            "page_content": text,
+                            "metadata": metadata,
+                        }
+                    )
+                return processed_results
+
+            search_kwargs = {"k": self.number_of_results}
+            search_type = self.search_type.lower()
+
+            if search_type == "similarity":
+                results = vector_store.similarity_search(query, **search_kwargs)
+                return [{"page_content": doc.page_content, "metadata": doc.metadata} for doc in results]
+            if search_type == "similarity_score_threshold":
+                search_kwargs["score_threshold"] = self.search_score_threshold
+                results = vector_store.similarity_search_with_relevance_scores(query, **search_kwargs)
+                return [
+                    {
+                        "page_content": doc.page_content,
+                        "metadata": doc.metadata,
+                        "score": score,
+                    }
+                    for doc, score in results
+                ]
+            if search_type == "mmr":
+                results = vector_store.max_marginal_relevance_search(query, **search_kwargs)
+                return [{"page_content": doc.page_content, "metadata": doc.metadata} for doc in results]
+
+        except Exception as e:
+            error_message = f"Error during search: {e}"
+            self.log(error_message)
+            raise RuntimeError(error_message) from e
+
+        error_message = f"Error during search. Invalid search type: {self.search_type}"
+        self.log(error_message)
+        raise ValueError(error_message)
+
+    def search_documents(self) -> list[Data]:
+        """Search for documents in the vector store based on the search input.
+
+        If no search input is provided, retrieve all documents.
+        """
+        try:
+            query = self.search_query.strip() if self.search_query else None
+            results = self.search(query)
+            retrieved_data = [
+                Data(
+                    file_path=result["metadata"].get("file_path", ""),
+                    text=result["page_content"],
+                )
+                for result in results
+            ]
+        except Exception as e:
+            error_message = f"Error during document search: {e}"
+            self.log(error_message)
+            raise RuntimeError(error_message) from e
+
+        self.status = retrieved_data
+        return retrieved_data

lfx/components/vectorstores/pgvector.py

@@ -0,0 +1,72 @@
+from langchain_community.vectorstores import PGVector
+
+from lfx.base.vectorstores.model import LCVectorStoreComponent, check_cached_vector_store
+from lfx.helpers.data import docs_to_data
+from lfx.io import HandleInput, IntInput, SecretStrInput, StrInput
+from lfx.schema.data import Data
+from lfx.utils.connection_string_parser import transform_connection_string
+
+
+class PGVectorStoreComponent(LCVectorStoreComponent):
+    display_name = "PGVector"
+    description = "PGVector Vector Store with search capabilities"
+    name = "pgvector"
+    icon = "cpu"
+
+    inputs = [
+        SecretStrInput(name="pg_server_url", display_name="PostgreSQL Server Connection String", required=True),
+        StrInput(name="collection_name", display_name="Table", required=True),
+        *LCVectorStoreComponent.inputs,
+        HandleInput(name="embedding", display_name="Embedding", input_types=["Embeddings"], required=True),
+        IntInput(
+            name="number_of_results",
+            display_name="Number of Results",
+            info="Number of results to return.",
+            value=4,
+            advanced=True,
+        ),
+    ]
+
+    @check_cached_vector_store
+    def build_vector_store(self) -> PGVector:
+        # Convert DataFrame to Data if needed using parent's method
+        self.ingest_data = self._prepare_ingest_data()
+
+        documents = []
+        for _input in self.ingest_data or []:
+            if isinstance(_input, Data):
+                documents.append(_input.to_lc_document())
+            else:
+                documents.append(_input)
+
+        connection_string_parsed = transform_connection_string(self.pg_server_url)
+
+        if documents:
+            pgvector = PGVector.from_documents(
+                embedding=self.embedding,
+                documents=documents,
+                collection_name=self.collection_name,
+                connection_string=connection_string_parsed,
+            )
+        else:
+            pgvector = PGVector.from_existing_index(
+                embedding=self.embedding,
+                collection_name=self.collection_name,
+                connection_string=connection_string_parsed,
+            )
+
+        return pgvector
+
+    def search_documents(self) -> list[Data]:
+        vector_store = self.build_vector_store()
+
+        if self.search_query and isinstance(self.search_query, str) and self.search_query.strip():
+            docs = vector_store.similarity_search(
+                query=self.search_query,
+                k=self.number_of_results,
+            )
+
+            data = docs_to_data(docs)
+            self.status = data
+            return data
+        return []

lfx/components/vectorstores/pinecone.py

@@ -0,0 +1,134 @@
+import numpy as np
+from langchain_core.vectorstores import VectorStore
+
+from lfx.base.vectorstores.model import LCVectorStoreComponent, check_cached_vector_store
+from lfx.helpers.data import docs_to_data
+from lfx.io import DropdownInput, HandleInput, IntInput, SecretStrInput, StrInput
+from lfx.schema.data import Data
+
+
+class PineconeVectorStoreComponent(LCVectorStoreComponent):
+    display_name = "Pinecone"
+    description = "Pinecone Vector Store with search capabilities"
+    name = "Pinecone"
+    icon = "Pinecone"
+    inputs = [
+        StrInput(name="index_name", display_name="Index Name", required=True),
+        StrInput(name="namespace", display_name="Namespace", info="Namespace for the index."),
+        DropdownInput(
+            name="distance_strategy",
+            display_name="Distance Strategy",
+            options=["Cosine", "Euclidean", "Dot Product"],
+            value="Cosine",
+            advanced=True,
+        ),
+        SecretStrInput(name="pinecone_api_key", display_name="Pinecone API Key", required=True),
+        StrInput(
+            name="text_key",
+            display_name="Text Key",
+            info="Key in the record to use as text.",
+            value="text",
+            advanced=True,
+        ),
+        *LCVectorStoreComponent.inputs,
+        HandleInput(name="embedding", display_name="Embedding", input_types=["Embeddings"]),
+        IntInput(
+            name="number_of_results",
+            display_name="Number of Results",
+            info="Number of results to return.",
+            value=4,
+            advanced=True,
+        ),
+    ]
+
+    @check_cached_vector_store
+    def build_vector_store(self) -> VectorStore:
+        """Build and return a Pinecone vector store instance."""
+        try:
+            from langchain_pinecone import PineconeVectorStore
+        except ImportError as e:
+            msg = "langchain-pinecone is not installed. Please install it with `pip install langchain-pinecone`."
+            raise ValueError(msg) from e
+
+        try:
+            from langchain_pinecone._utilities import DistanceStrategy
+
+            # Wrap the embedding model to ensure float32 output
+            wrapped_embeddings = Float32Embeddings(self.embedding)
+
+            # Convert distance strategy
+            distance_strategy = self.distance_strategy.replace(" ", "_").upper()
+            distance_strategy = DistanceStrategy[distance_strategy]
+
+            # Initialize Pinecone instance with wrapped embeddings
+            pinecone = PineconeVectorStore(
+                index_name=self.index_name,
+                embedding=wrapped_embeddings,  # Use wrapped embeddings
+                text_key=self.text_key,
+                namespace=self.namespace,
+                distance_strategy=distance_strategy,
+                pinecone_api_key=self.pinecone_api_key,
+            )
+        except Exception as e:
+            error_msg = "Error building Pinecone vector store"
+            raise ValueError(error_msg) from e
+        else:
+            self.ingest_data = self._prepare_ingest_data()
+
+            # Process documents if any
+            documents = []
+            if self.ingest_data:
+                # Convert DataFrame to Data if needed using parent's method
+
+                for doc in self.ingest_data:
+                    if isinstance(doc, Data):
+                        documents.append(doc.to_lc_document())
+                    else:
+                        documents.append(doc)
+
+                if documents:
+                    pinecone.add_documents(documents)
+
+            return pinecone
+
+    def search_documents(self) -> list[Data]:
+        """Search documents in the vector store."""
+        try:
+            if not self.search_query or not isinstance(self.search_query, str) or not self.search_query.strip():
+                return []
+
+            vector_store = self.build_vector_store()
+            docs = vector_store.similarity_search(
+                query=self.search_query,
+                k=self.number_of_results,
+            )
+        except Exception as e:
+            error_msg = "Error searching documents"
+            raise ValueError(error_msg) from e
+        else:
+            data = docs_to_data(docs)
+            self.status = data
+            return data
+
+
+class Float32Embeddings:
+    """Wrapper class to ensure float32 embeddings."""
+
+    def __init__(self, base_embeddings):
+        self.base_embeddings = base_embeddings
+
+    def embed_documents(self, texts):
+        embeddings = self.base_embeddings.embed_documents(texts)
+        if isinstance(embeddings, np.ndarray):
+            return [[self._force_float32(x) for x in vec] for vec in embeddings]
+        return [[self._force_float32(x) for x in vec] for vec in embeddings]
+
+    def embed_query(self, text):
+        embedding = self.base_embeddings.embed_query(text)
+        if isinstance(embedding, np.ndarray):
+            return [self._force_float32(x) for x in embedding]
+        return [self._force_float32(x) for x in embedding]
+
+    def _force_float32(self, value):
+        """Convert any numeric type to Python float."""
+        return float(np.float32(value))