lfx-nightly 0.1.12.dev42__py3-none-any.whl → 0.2.0.dev0__py3-none-any.whl

lfx/components/vectorstores/hcd.py

@@ -1,314 +0,0 @@
-from lfx.base.vectorstores.model import LCVectorStoreComponent, check_cached_vector_store
-from lfx.helpers.data import docs_to_data
-from lfx.inputs.inputs import DictInput, FloatInput
-from lfx.io import (
-    BoolInput,
-    DropdownInput,
-    HandleInput,
-    IntInput,
-    MultilineInput,
-    SecretStrInput,
-    StrInput,
-)
-from lfx.schema.data import Data
-
-
-class HCDVectorStoreComponent(LCVectorStoreComponent):
-    display_name: str = "Hyper-Converged Database"
-    description: str = "Implementation of Vector Store using Hyper-Converged Database (HCD) with search capabilities"
-    name = "HCD"
-    icon: str = "HCD"
-
-    inputs = [
-        StrInput(
-            name="collection_name",
-            display_name="Collection Name",
-            info="The name of the collection within HCD where the vectors will be stored.",
-            required=True,
-        ),
-        StrInput(
-            name="username",
-            display_name="HCD Username",
-            info="Authentication username for accessing HCD.",
-            value="hcd-superuser",
-            required=True,
-        ),
-        SecretStrInput(
-            name="password",
-            display_name="HCD Password",
-            info="Authentication password for accessing HCD.",
-            value="HCD_PASSWORD",
-            required=True,
-        ),
-        SecretStrInput(
-            name="api_endpoint",
-            display_name="HCD API Endpoint",
-            info="API endpoint URL for the HCD service.",
-            value="HCD_API_ENDPOINT",
-            required=True,
-        ),
-        *LCVectorStoreComponent.inputs,
-        StrInput(
-            name="namespace",
-            display_name="Namespace",
-            info="Optional namespace within HCD to use for the collection.",
-            value="default_namespace",
-            advanced=True,
-        ),
-        MultilineInput(
-            name="ca_certificate",
-            display_name="CA Certificate",
-            info="Optional CA certificate for TLS connections to HCD.",
-            advanced=True,
-        ),
-        DropdownInput(
-            name="metric",
-            display_name="Metric",
-            info="Optional distance metric for vector comparisons in the vector store.",
-            options=["cosine", "dot_product", "euclidean"],
-            advanced=True,
-        ),
-        IntInput(
-            name="batch_size",
-            display_name="Batch Size",
-            info="Optional number of data to process in a single batch.",
-            advanced=True,
-        ),
-        IntInput(
-            name="bulk_insert_batch_concurrency",
-            display_name="Bulk Insert Batch Concurrency",
-            info="Optional concurrency level for bulk insert operations.",
-            advanced=True,
-        ),
-        IntInput(
-            name="bulk_insert_overwrite_concurrency",
-            display_name="Bulk Insert Overwrite Concurrency",
-            info="Optional concurrency level for bulk insert operations that overwrite existing data.",
-            advanced=True,
-        ),
-        IntInput(
-            name="bulk_delete_concurrency",
-            display_name="Bulk Delete Concurrency",
-            info="Optional concurrency level for bulk delete operations.",
-            advanced=True,
-        ),
-        DropdownInput(
-            name="setup_mode",
-            display_name="Setup Mode",
-            info="Configuration mode for setting up the vector store, with options like 'Sync', 'Async', or 'Off'.",
-            options=["Sync", "Async", "Off"],
-            advanced=True,
-            value="Sync",
-        ),
-        BoolInput(
-            name="pre_delete_collection",
-            display_name="Pre Delete Collection",
-            info="Boolean flag to determine whether to delete the collection before creating a new one.",
-            advanced=True,
-        ),
-        StrInput(
-            name="metadata_indexing_include",
-            display_name="Metadata Indexing Include",
-            info="Optional list of metadata fields to include in the indexing.",
-            advanced=True,
-        ),
-        HandleInput(
-            name="embedding",
-            display_name="Embedding or Astra Vectorize",
-            input_types=["Embeddings", "dict"],
-            # TODO: This should be optional, but need to refactor langchain-astradb first.
-            info="Allows either an embedding model or an Astra Vectorize configuration.",
-        ),
-        StrInput(
-            name="metadata_indexing_exclude",
-            display_name="Metadata Indexing Exclude",
-            info="Optional list of metadata fields to exclude from the indexing.",
-            advanced=True,
-        ),
-        StrInput(
-            name="collection_indexing_policy",
-            display_name="Collection Indexing Policy",
-            info="Optional dictionary defining the indexing policy for the collection.",
-            advanced=True,
-        ),
-        IntInput(
-            name="number_of_results",
-            display_name="Number of Results",
-            info="Number of results to return.",
-            advanced=True,
-            value=4,
-        ),
-        DropdownInput(
-            name="search_type",
-            display_name="Search Type",
-            info="Search type to use",
-            options=["Similarity", "Similarity with score threshold", "MMR (Max Marginal Relevance)"],
-            value="Similarity",
-            advanced=True,
-        ),
-        FloatInput(
-            name="search_score_threshold",
-            display_name="Search Score Threshold",
-            info="Minimum similarity score threshold for search results. "
-            "(when using 'Similarity with score threshold')",
-            value=0,
-            advanced=True,
-        ),
-        DictInput(
-            name="search_filter",
-            display_name="Search Metadata Filter",
-            info="Optional dictionary of filters to apply to the search query.",
-            advanced=True,
-            is_list=True,
-        ),
-    ]
-
-    @check_cached_vector_store
-    def build_vector_store(self):
-        try:
-            from langchain_astradb import AstraDBVectorStore
-            from langchain_astradb.utils.astradb import SetupMode
-        except ImportError as e:
-            msg = (
-                "Could not import langchain Astra DB integration package. "
-                "Please install it with `pip install langchain-astradb`."
-            )
-            raise ImportError(msg) from e
-
-        try:
-            from astrapy.authentication import UsernamePasswordTokenProvider
-            from astrapy.constants import Environment
-        except ImportError as e:
-            msg = "Could not import astrapy integration package. Please install it with `pip install astrapy`."
-            raise ImportError(msg) from e
-
-        try:
-            if not self.setup_mode:
-                self.setup_mode = self._inputs["setup_mode"].options[0]
-
-            setup_mode_value = SetupMode[self.setup_mode.upper()]
-        except KeyError as e:
-            msg = f"Invalid setup mode: {self.setup_mode}"
-            raise ValueError(msg) from e
-
-        if not isinstance(self.embedding, dict):
-            embedding_dict = {"embedding": self.embedding}
-        else:
-            from astrapy.info import VectorServiceOptions
-
-            dict_options = self.embedding.get("collection_vector_service_options", {})
-            dict_options["authentication"] = {
-                k: v for k, v in dict_options.get("authentication", {}).items() if k and v
-            }
-            dict_options["parameters"] = {k: v for k, v in dict_options.get("parameters", {}).items() if k and v}
-            embedding_dict = {"collection_vector_service_options": VectorServiceOptions.from_dict(dict_options)}
-            collection_embedding_api_key = self.embedding.get("collection_embedding_api_key")
-            if collection_embedding_api_key:
-                embedding_dict["collection_embedding_api_key"] = collection_embedding_api_key
-
-        token_provider = UsernamePasswordTokenProvider(self.username, self.password)
-        vector_store_kwargs = {
-            **embedding_dict,
-            "collection_name": self.collection_name,
-            "token": token_provider,
-            "api_endpoint": self.api_endpoint,
-            "namespace": self.namespace,
-            "metric": self.metric or None,
-            "batch_size": self.batch_size or None,
-            "bulk_insert_batch_concurrency": self.bulk_insert_batch_concurrency or None,
-            "bulk_insert_overwrite_concurrency": self.bulk_insert_overwrite_concurrency or None,
-            "bulk_delete_concurrency": self.bulk_delete_concurrency or None,
-            "setup_mode": setup_mode_value,
-            "pre_delete_collection": self.pre_delete_collection or False,
-            "environment": Environment.HCD,
-        }
-
-        if self.metadata_indexing_include:
-            vector_store_kwargs["metadata_indexing_include"] = self.metadata_indexing_include
-        elif self.metadata_indexing_exclude:
-            vector_store_kwargs["metadata_indexing_exclude"] = self.metadata_indexing_exclude
-        elif self.collection_indexing_policy:
-            vector_store_kwargs["collection_indexing_policy"] = self.collection_indexing_policy
-
-        try:
-            vector_store = AstraDBVectorStore(**vector_store_kwargs)
-        except Exception as e:
-            msg = f"Error initializing AstraDBVectorStore: {e}"
-            raise ValueError(msg) from e
-
-        self._add_documents_to_vector_store(vector_store)
-        return vector_store
-
-    def _add_documents_to_vector_store(self, vector_store) -> None:
-        # Convert DataFrame to Data if needed using parent's method
-        self.ingest_data = self._prepare_ingest_data()
-
-        documents = []
-        for _input in self.ingest_data or []:
-            if isinstance(_input, Data):
-                documents.append(_input.to_lc_document())
-            else:
-                msg = "Vector Store Inputs must be Data objects."
-                raise TypeError(msg)
-
-        if documents:
-            self.log(f"Adding {len(documents)} documents to the Vector Store.")
-            try:
-                vector_store.add_documents(documents)
-            except Exception as e:
-                msg = f"Error adding documents to AstraDBVectorStore: {e}"
-                raise ValueError(msg) from e
-        else:
-            self.log("No documents to add to the Vector Store.")
-
-    def _map_search_type(self) -> str:
-        if self.search_type == "Similarity with score threshold":
-            return "similarity_score_threshold"
-        if self.search_type == "MMR (Max Marginal Relevance)":
-            return "mmr"
-        return "similarity"
-
-    def _build_search_args(self):
-        args = {
-            "k": self.number_of_results,
-            "score_threshold": self.search_score_threshold,
-        }
-
-        if self.search_filter:
-            clean_filter = {k: v for k, v in self.search_filter.items() if k and v}
-            if len(clean_filter) > 0:
-                args["filter"] = clean_filter
-        return args
-
-    def search_documents(self) -> list[Data]:
-        vector_store = self.build_vector_store()
-
-        self.log(f"Search query: {self.search_query}")
-        self.log(f"Search type: {self.search_type}")
-        self.log(f"Number of results: {self.number_of_results}")
-
-        if self.search_query and isinstance(self.search_query, str) and self.search_query.strip():
-            try:
-                search_type = self._map_search_type()
-                search_args = self._build_search_args()
-
-                docs = vector_store.search(query=self.search_query, search_type=search_type, **search_args)
-            except Exception as e:
-                msg = f"Error performing search in AstraDBVectorStore: {e}"
-                raise ValueError(msg) from e
-
-            self.log(f"Retrieved documents: {len(docs)}")
-
-            data = docs_to_data(docs)
-            self.log(f"Converted documents to data: {len(data)}")
-            self.status = data
-            return data
-        self.log("No search input provided. Skipping search.")
-        return []
-
-    def get_retriever_kwargs(self):
-        search_args = self._build_search_args()
-        return {
-            "search_type": self._map_search_type(),
-            "search_kwargs": search_args,
-        }

lfx/components/vectorstores/milvus.py

@@ -1,115 +0,0 @@
-from lfx.base.vectorstores.model import LCVectorStoreComponent, check_cached_vector_store
-from lfx.helpers.data import docs_to_data
-from lfx.io import (
-    BoolInput,
-    DictInput,
-    DropdownInput,
-    FloatInput,
-    HandleInput,
-    IntInput,
-    SecretStrInput,
-    StrInput,
-)
-from lfx.schema.data import Data
-
-
-class MilvusVectorStoreComponent(LCVectorStoreComponent):
-    """Milvus vector store with search capabilities."""
-
-    display_name: str = "Milvus"
-    description: str = "Milvus vector store with search capabilities"
-    name = "Milvus"
-    icon = "Milvus"
-
-    inputs = [
-        StrInput(name="collection_name", display_name="Collection Name", value="langflow"),
-        StrInput(name="collection_description", display_name="Collection Description", value=""),
-        StrInput(
-            name="uri",
-            display_name="Connection URI",
-            value="http://localhost:19530",
-        ),
-        SecretStrInput(
-            name="password",
-            display_name="Token",
-            value="",
-            info="Ignore this field if no token is required to make connection.",
-        ),
-        DictInput(name="connection_args", display_name="Other Connection Arguments", advanced=True),
-        StrInput(name="primary_field", display_name="Primary Field Name", value="pk"),
-        StrInput(name="text_field", display_name="Text Field Name", value="text"),
-        StrInput(name="vector_field", display_name="Vector Field Name", value="vector"),
-        DropdownInput(
-            name="consistency_level",
-            display_name="Consistencey Level",
-            options=["Bounded", "Session", "Strong", "Eventual"],
-            value="Session",
-            advanced=True,
-        ),
-        DictInput(name="index_params", display_name="Index Parameters", advanced=True),
-        DictInput(name="search_params", display_name="Search Parameters", advanced=True),
-        BoolInput(name="drop_old", display_name="Drop Old Collection", value=False, advanced=True),
-        FloatInput(name="timeout", display_name="Timeout", advanced=True),
-        *LCVectorStoreComponent.inputs,
-        HandleInput(name="embedding", display_name="Embedding", input_types=["Embeddings"]),
-        IntInput(
-            name="number_of_results",
-            display_name="Number of Results",
-            info="Number of results to return.",
-            value=4,
-            advanced=True,
-        ),
-    ]
-
-    @check_cached_vector_store
-    def build_vector_store(self):
-        try:
-            from langchain_milvus.vectorstores import Milvus as LangchainMilvus
-        except ImportError as e:
-            msg = "Could not import Milvus integration package. Please install it with `pip install langchain-milvus`."
-            raise ImportError(msg) from e
-        self.connection_args.update(uri=self.uri, token=self.password)
-        milvus_store = LangchainMilvus(
-            embedding_function=self.embedding,
-            collection_name=self.collection_name,
-            collection_description=self.collection_description,
-            connection_args=self.connection_args,
-            consistency_level=self.consistency_level,
-            index_params=self.index_params,
-            search_params=self.search_params,
-            drop_old=self.drop_old,
-            auto_id=True,
-            primary_field=self.primary_field,
-            text_field=self.text_field,
-            vector_field=self.vector_field,
-            timeout=self.timeout,
-        )
-
-        # Convert DataFrame to Data if needed using parent's method
-        self.ingest_data = self._prepare_ingest_data()
-
-        documents = []
-        for _input in self.ingest_data or []:
-            if isinstance(_input, Data):
-                documents.append(_input.to_lc_document())
-            else:
-                documents.append(_input)
-
-        if documents:
-            milvus_store.add_documents(documents)
-
-        return milvus_store
-
-    def search_documents(self) -> list[Data]:
-        vector_store = self.build_vector_store()
-
-        if self.search_query and isinstance(self.search_query, str) and self.search_query.strip():
-            docs = vector_store.similarity_search(
-                query=self.search_query,
-                k=self.number_of_results,
-            )
-
-            data = docs_to_data(docs)
-            self.status = data
-            return data
-        return []

lfx/components/vectorstores/mongodb_atlas.py

@@ -1,213 +0,0 @@
-import tempfile
-import time
-
-import certifi
-from langchain_community.vectorstores import MongoDBAtlasVectorSearch
-from pymongo.collection import Collection
-from pymongo.operations import SearchIndexModel
-
-from lfx.base.vectorstores.model import LCVectorStoreComponent, check_cached_vector_store
-from lfx.helpers.data import docs_to_data
-from lfx.io import BoolInput, DropdownInput, HandleInput, IntInput, SecretStrInput, StrInput
-from lfx.schema.data import Data
-
-
-class MongoVectorStoreComponent(LCVectorStoreComponent):
-    display_name = "MongoDB Atlas"
-    description = "MongoDB Atlas Vector Store with search capabilities"
-    name = "MongoDBAtlasVector"
-    icon = "MongoDB"
-    INSERT_MODES = ["append", "overwrite"]
-    SIMILARITY_OPTIONS = ["cosine", "euclidean", "dotProduct"]
-    QUANTIZATION_OPTIONS = ["scalar", "binary"]
-    inputs = [
-        SecretStrInput(name="mongodb_atlas_cluster_uri", display_name="MongoDB Atlas Cluster URI", required=True),
-        BoolInput(name="enable_mtls", display_name="Enable mTLS", value=False, advanced=True, required=True),
-        SecretStrInput(
-            name="mongodb_atlas_client_cert",
-            display_name="MongoDB Atlas Combined Client Certificate",
-            required=False,
-            info="Client Certificate combined with the private key in the following format:\n "
-            "-----BEGIN PRIVATE KEY-----\n...\n -----END PRIVATE KEY-----\n-----BEGIN CERTIFICATE-----\n"
-            "...\n-----END CERTIFICATE-----\n",
-        ),
-        StrInput(name="db_name", display_name="Database Name", required=True),
-        StrInput(name="collection_name", display_name="Collection Name", required=True),
-        StrInput(
-            name="index_name",
-            display_name="Index Name",
-            required=True,
-            info="The name of Atlas Search index, it should be a Vector Search.",
-        ),
-        *LCVectorStoreComponent.inputs,
-        DropdownInput(
-            name="insert_mode",
-            display_name="Insert Mode",
-            options=INSERT_MODES,
-            value=INSERT_MODES[0],
-            info="How to insert new documents into the collection.",
-            advanced=True,
-        ),
-        HandleInput(name="embedding", display_name="Embedding", input_types=["Embeddings"]),
-        IntInput(
-            name="number_of_results",
-            display_name="Number of Results",
-            info="Number of results to return.",
-            value=4,
-            advanced=True,
-        ),
-        StrInput(
-            name="index_field",
-            display_name="Index Field",
-            advanced=True,
-            required=True,
-            info="The field to index.",
-            value="embedding",
-        ),
-        StrInput(
-            name="filter_field", display_name="Filter Field", advanced=True, info="The field to filter the index."
-        ),
-        IntInput(
-            name="number_dimensions",
-            display_name="Number of Dimensions",
-            info="Embedding Context Length.",
-            value=1536,
-            advanced=True,
-            required=True,
-        ),
-        DropdownInput(
-            name="similarity",
-            display_name="Similarity",
-            options=SIMILARITY_OPTIONS,
-            value=SIMILARITY_OPTIONS[0],
-            info="The method used to measure the similarity between vectors.",
-            advanced=True,
-        ),
-        DropdownInput(
-            name="quantization",
-            display_name="Quantization",
-            options=QUANTIZATION_OPTIONS,
-            value=None,
-            info="Quantization reduces memory costs converting 32-bit floats to smaller data types",
-            advanced=True,
-        ),
-    ]
-
-    @check_cached_vector_store
-    def build_vector_store(self) -> MongoDBAtlasVectorSearch:
-        try:
-            from pymongo import MongoClient
-        except ImportError as e:
-            msg = "Please install pymongo to use MongoDB Atlas Vector Store"
-            raise ImportError(msg) from e
-
-        # Create temporary files for the client certificate
-        if self.enable_mtls:
-            client_cert_path = None
-            try:
-                client_cert = self.mongodb_atlas_client_cert.replace(" ", "\n")
-                client_cert = client_cert.replace("-----BEGIN\nPRIVATE\nKEY-----", "-----BEGIN PRIVATE KEY-----")
-                client_cert = client_cert.replace(
-                    "-----END\nPRIVATE\nKEY-----\n-----BEGIN\nCERTIFICATE-----",
-                    "-----END PRIVATE KEY-----\n-----BEGIN CERTIFICATE-----",
-                )
-                client_cert = client_cert.replace("-----END\nCERTIFICATE-----", "-----END CERTIFICATE-----")
-                with tempfile.NamedTemporaryFile(delete=False) as client_cert_file:
-                    client_cert_file.write(client_cert.encode("utf-8"))
-                    client_cert_path = client_cert_file.name
-
-            except Exception as e:
-                msg = f"Failed to write certificate to temporary file: {e}"
-                raise ValueError(msg) from e
-
-        try:
-            mongo_client: MongoClient = (
-                MongoClient(
-                    self.mongodb_atlas_cluster_uri,
-                    tls=True,
-                    tlsCertificateKeyFile=client_cert_path,
-                    tlsCAFile=certifi.where(),
-                )
-                if self.enable_mtls
-                else MongoClient(self.mongodb_atlas_cluster_uri)
-            )
-
-            collection = mongo_client[self.db_name][self.collection_name]
-
-        except Exception as e:
-            msg = f"Failed to connect to MongoDB Atlas: {e}"
-            raise ValueError(msg) from e
-
-        # Convert DataFrame to Data if needed using parent's method
-        self.ingest_data = self._prepare_ingest_data()
-
-        documents = []
-        for _input in self.ingest_data or []:
-            if isinstance(_input, Data):
-                documents.append(_input.to_lc_document())
-            else:
-                documents.append(_input)
-
-        if documents:
-            self.__insert_mode(collection)
-
-            return MongoDBAtlasVectorSearch.from_documents(
-                documents=documents, embedding=self.embedding, collection=collection, index_name=self.index_name
-            )
-        return MongoDBAtlasVectorSearch(embedding=self.embedding, collection=collection, index_name=self.index_name)
-
-    def search_documents(self) -> list[Data]:
-        from bson.objectid import ObjectId
-
-        vector_store = self.build_vector_store()
-
-        self.verify_search_index(vector_store._collection)
-
-        if self.search_query and isinstance(self.search_query, str):
-            docs = vector_store.similarity_search(
-                query=self.search_query,
-                k=self.number_of_results,
-            )
-            for doc in docs:
-                doc.metadata = {
-                    key: str(value) if isinstance(value, ObjectId) else value for key, value in doc.metadata.items()
-                }
-
-            data = docs_to_data(docs)
-            self.status = data
-            return data
-        return []
-
-    def __insert_mode(self, collection: Collection) -> None:
-        if self.insert_mode == "overwrite":
-            collection.delete_many({})  # Delete all documents while preserving collection structure
-
-    def verify_search_index(self, collection: Collection) -> None:
-        """Verify if the search index exists, if not, create it.
-
-        Args:
-            collection (Collection): The collection to verify the search index on.
-        """
-        indexes = collection.list_search_indexes()
-
-        index_names_types = {idx["name"]: idx["type"] for idx in indexes}
-        index_names = list(index_names_types.keys())
-        index_type = index_names_types.get(self.index_name)
-        if self.index_name not in index_names and index_type != "vectorSearch":
-            collection.create_search_index(self.__create_index_definition())
-
-            time.sleep(20)  # Give some time for index to be ready
-
-    def __create_index_definition(self) -> SearchIndexModel:
-        fields = [
-            {
-                "type": "vector",
-                "path": self.index_field,
-                "numDimensions": self.number_dimensions,
-                "similarity": self.similarity,
-                "quantization": self.quantization,
-            }
-        ]
-        if self.filter_field:
-            fields.append({"type": "filter", "path": self.filter_field})
-        return SearchIndexModel(definition={"fields": fields}, name=self.index_name, type="vectorSearch")